[rocpd] Adding merge and package submodules for rocpd

source/lib/python/rocpd/__main__.py

@@ -39,8 +39,10 @@ def main(argv=None, config=None):
     """
     import argparse
     from . import csv
+    from . import merge
     from . import otf2
     from . import output_config
+    from . import package
     from . import pftrace
     from . import query
     from . import summary
@@ -64,6 +66,20 @@ def main(argv=None, config=None):
     Convert 2 databases, output CSV, OTF2, and perfetto trace formats
     $ rocpd convert -i db{3,4}.db --output-format csv otf2 pftrace
 
+"""
+
+    merge_examples = """
+
+Example usage:
+
+    TODO: Add examples for merge command
+"""
+
+    package_examples = """
+
+Example usage:
+
+    TODO: Add examples for package command
 """
 
     query_examples = """
@@ -118,6 +134,22 @@ def main(argv=None, config=None):
         epilog=convert_examples,
     )
 
+    merger = subparsers.add_parser(
+        "merge",
+        description="Generate merged database from rocPD databases",
+        allow_abbrev=False,
+        formatter_class=argparse.RawTextHelpFormatter,
+        epilog=merge_examples,
+    )
+
+    packager = subparsers.add_parser(
+        "package",
+        description="Package database files into .rpdb output",
+        allow_abbrev=False,
+        formatter_class=argparse.RawTextHelpFormatter,
+        epilog=package_examples,
+    )
+
     query_reporter = subparsers.add_parser(
         "query",
         description="Generate output on a query",
@@ -158,6 +190,26 @@ def get_output_type(val):
         required=True,
     )
 
+    merger_required_params = merger.add_argument_group("Required options")
+    merger_required_params.add_argument(
+        "-i",
+        "--input",
+        required=True,
+        type=output_config.check_file_exists,
+        nargs="+",
+        help="Input path and filename to one or more database(s)",
+    )
+
+    packager_required_params = packager.add_argument_group("Required options")
+    packager_required_params.add_argument(
+        "-i",
+        "--input",
+        required=True,
+        type=output_config.check_file_exists,
+        nargs="+",
+        help="Input path and filename to one or more database(s)",
+    )
+
     query_required_params = query_reporter.add_argument_group("Required options")
     query_required_params.add_argument(
         "-i",
@@ -186,6 +238,12 @@ def get_output_type(val):
     valid_otf2_args = otf2.add_args(converter)
     valid_time_window_args = time_window.add_args(converter)
 
+    # merge: subparser args
+    valid_merge_args = merge.add_args(merger)
+
+    # package: subparser args
+    valid_package_args = package.add_args(packager)
+
     # query: subparser args
     valid_out_config_args = output_config.add_args(query_reporter)
     valid_query_args = query.add_args(query_reporter)
@@ -211,6 +269,17 @@ def get_output_type(val):
         parser.print_help()
         return
 
+    # convert to real number of DB input files
+    input_files = package.flatten_rocpd_yaml_input_file(args.input)
+    db_count = len(input_files)
+
+    # TODO: add logic to determine how many DBs to merge into
+    ## SQLITE_MAX_ATTACHED == 10, can query once you have connection
+    ## if db_count > 10
+    ##  call merge to combine to fewer DBs
+    ##  optionally, can also package up into own .rpdb package
+    ## Only after DBs have been merged and < SQLITE_MAX_ATTACHED, then we can call importer to attach DBs for analysis/convert
+
     # if the user requested converter, process the conversion
     if args.command == "convert":
         # process the args
@@ -224,7 +293,7 @@ def get_output_type(val):
         window_args = time_window.process_args(args, valid_time_window_args)
 
         # now start processing the data.  Import the data and merge the views
-        importData = RocpdImportData(args.input)
+        importData = RocpdImportData(input_files)
 
         # adjust the time window view of the data
         if window_args is not None:
@@ -258,6 +327,18 @@ def get_output_type(val):
             else:
                 print(f"Warning: Unsupported output format '{out_format}'")
 
+    # if the user requested merge module, execute the merge
+    elif args.command == "merge":
+        # merge subparser args
+        merge_args = merge.process_args(args, valid_merge_args)
+        merge.execute(input_files, **merge_args)
+
+    # if the user requested package module, package up the database
+    elif args.command == "package":
+        # merge subparser args
+        package_args = package.process_args(args, valid_package_args)
+        package.execute(input_files, **package_args)
+
     # if the user requested query module, execute the query
     elif args.command == "query":
         # query subparser args
@@ -268,7 +349,7 @@ def get_output_type(val):
         all_args = {**query_args, **out_cfg_args}
 
         query.execute(
-            args.input,
+            input_files,
             args,
             window_args=window_args,
             **all_args,
@@ -282,7 +363,7 @@ def get_output_type(val):
         window_args = time_window.process_args(args, valid_time_window_args)
 
         # now start processing the data.  Import the data and merge the views
-        importData = RocpdImportData(args.input)
+        importData = RocpdImportData(input_files)
 
         # adjust the time window view of the data
         if window_args is not None: