675.force compile option

fre/__init__.py

@@ -9,7 +9,7 @@
 
 fre_logger = logging.getLogger(__name__)
 
-FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s"
+FORMAT = "[%(levelname)7s:%(filename)25s:%(funcName)24s] %(message)s"
 logging.basicConfig(level = logging.WARNING,
                     format = FORMAT,
                     filename = None,

fre/make/create_compile_script.py

@@ -34,9 +34,41 @@
 
 fre_logger = logging.getLogger(__name__)
 
-def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejobs: int = 4,
+def compile_call(fremake_yaml, template_path, src_dir, bld_dir, target, platform, jobs):
+    """
+    lkjsdlfkjs
+
+    :param fremake_yaml:
+    :type fremake_yaml:
+    :param template_path:
+    :type template_path:
+    :param src_dir:
+    :type src_dir:
+    :param bld_dir:
+    :type bld_dir:
+    :param target:
+    :type target:
+    :param platform:
+    :type platform:
+    :param jobs:
+    :type jobs:
+    """
+    fremake_build = buildBaremetal.buildBaremetal(exp=fremake_yaml["experiment"],
+                                                 mkTemplatePath=template_path,
+                                                 srcDir=src_dir,
+                                                 bldDir=bld_dir,
+                                                 target=target,
+                                                 env_setup=platform["envSetup"],
+                                                 jobs=jobs)
+    for c in fremake_yaml['src']:
+        fremake_build.writeBuildComponents(c)
+    fremake_build.writeScript()
+    fre_logger.info("Compile script created: %s/compile.sh", bld_dir)
+
+def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4,
                    nparallel: int = 1, execute: Optional[bool] = False,
-                   verbose: Optional[bool] = None):
+                   verbose: Optional[bool] = False,
+                   force_compile: Optional[bool] = False):
     """
     This function compile_create generates the compile script for bare-metal build.
 
@@ -55,6 +87,8 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob
     :type execute: bool
     :param verbose: If True, increase verbosity output
     :type verbose: bool
+    :param force_compile: Re-create compile script if specified
+    :type force_compile: bool
     :raises ValueError:
         - Error if platform does not exist in platforms yaml configuration 
         - Error if the mkmf template defined in platforms yaml does not exist
@@ -91,6 +125,7 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob
     model_yaml = yamlfre.freyaml(full_combined, fre_vars)
     fremake_yaml = model_yaml.getCompileYaml()
 
+    tlist = target
     ## Error checking the targets
     for target_name in tlist:
         target = targetfre.fretarget(target_name)
@@ -117,21 +152,58 @@ def compile_create(yamlfile:str, platform:tuple[str], target:tuple[str], makejob
                                                     model_root = platform["modelRoot"],
                                                     container_flag = platform["container"])
 
-                ## Create a list of compile scripts to run in parallel
-                fremake_build = buildBaremetal.buildBaremetal(exp=fremake_yaml["experiment"],
-                                                             mkTemplatePath=template_path,
-                                                             srcDir=src_dir,
-                                                             bldDir=bld_dir,
-                                                             target=target,
-                                                             env_setup=platform["envSetup"],
-                                                             jobs=jobs)
-                for c in fremake_yaml['src']:
-                    fremake_build.writeBuildComponents(c)
-                fremake_build.writeScript()
-                fremake_build_list.append(fremake_build)
-                fre_logger.info("Compile script created here: %s/compile.sh", bld_dir)
-
+                if not Path(f"{bld_dir}/compile.sh").exists():
+                    ## Create a list of compile scripts to run in parallel
+                    compile_call(fremake_yaml = fremake_yaml,
+                                 template_path = template_path,
+                                 src_dir = src_dir,
+                                 bld_dir = bld_dir,
+                                 target = target,
+                                 platform = platform,
+                                 jobs = jobs)
+                    fremake_build_list.append(f"{bld_dir}/compile.sh")
+                elif Path(f"{bld_dir}/compile.sh").exists() and force_compile:
+                    # Remove old compile script
+                    fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir)
+                    fre_logger.warning("*** REMOVING COMPILE SCRIPT ***")
+                    Path(f"{bld_dir}/compile.sh").unlink()
+
+                    # Re-create compile script
+                    compile_call(fremake_yaml = fremake_yaml,
+                                 template_path = template_path,
+                                 src_dir = src_dir,
+                                 bld_dir = bld_dir,
+                                 target = target,
+                                 platform = platform,
+                                 jobs = jobs)
+
+                    fremake_build_list.append(f"{bld_dir}/compile.sh")
+                elif Path(f"{bld_dir}/compile.sh").exists() and not force_compile:
+                    fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir)
+                    fremake_build_list.append(f"{bld_dir}/compile.sh")
+                    ###COMPARE THE TWO TO SEE IF IT'S CHANGED###--> filecmp or difflib
+                    ###IF CHANGED, THROW ERROR###
+                    ###SHOULD IT ALSO BE RE-CREATED IF CHECKOUT RE-CREATED?? -->
+                    ### I THINK THIS WILL BE FOR "ALL" SUBTOOL###
+
+    fre_logger.setLevel(level=logging.INFO)
+    fre_logger.info("Compile scripts available/generated with specified platform-target combination: ")
+    for i in fremake_build_list:
+        fre_logger.info("  - %s", i)
+    fre_logger.setLevel(level=logging.WARNING)
+
+    # Returns the exit status for multiprocessing pool command
     if execute:
         if baremetal_run:
-            pool = Pool(processes=nparallel)  # Create a multiprocessing Pool
-            pool.map(buildBaremetal.fremake_parallel, fremake_build_list)  # process data_inputs iterable with pool
+            # Create a multiprocessing Pool
+            pool = Pool(processes=nparallel)
+            # process data_inputs iterable with pool
+            results = pool.map(buildBaremetal.fremake_parallel, fremake_build_list)
+
+            for r in results:
+                for key,value in r.items():
+                    if key == 1:
+                        fre_logger.error("ERROR: compile NOT successful")
+                        fre_logger.error("Check the generated log: %s", value)
+                    elif key == 0:
+                        fre_logger.info("Compile successful")

fre/make/create_makefile_script.py

@@ -142,3 +142,4 @@ def makefile_create(yamlfile: str, platform: tuple[str], target: tuple[str]):
                 fre_logger.setLevel(logging.INFO)
                 fre_logger.info("Makefile created: %s/Makefile", tmp_dir)
                 fre_logger.setLevel(former_log_level)
+    fre_logger.warning("")

fre/make/fremake.py

@@ -109,19 +109,22 @@ def make_cli():
               is_flag = True,
               default = False, 
               help = "Force a git checkout if the source directory already exists.")
+@click.option("--force-compile",
+              is_flag = True,
+              help = "Re-create the compile script in case it exists already.")
 @click.option("-v",
               "--verbose",
               is_flag = True,
               help = _VERBOSE_OPT_HELP)
 def all(yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_checkout, no_format_transfer, execute,
-        verbose, force_checkout):
+        verbose, force_checkout, force_compile):
     """
     - Perform all fre make functions; for baremetal platforms: create checkout script, makefile, and compile scripts;
     for container platforms: create checkout script, makefile, Dockerfile, and createContainer script
     """
     run_fremake_script.fremake_run(
         yamlfile, platform, target, nparallel, makejobs, gitjobs, no_parallel_checkout, no_format_transfer, execute,
-        verbose, force_checkout)
+        verbose, force_checkout, force_compile)
 
 @make_cli.command('checkout-script')
 @click.option("-y",
@@ -216,14 +219,17 @@ def makefile(yamlfile, platform, target):
               default = False,
               help = """Execute the compile script immediately following its generation.
               The default behavior is to generate the script, but not execute.""")
+@click.option("--force-compile",
+              is_flag = True,
+              help = "Re-create the compile script in case it exists already.")
 @click.option("-v",
               "--verbose",
               is_flag = True,
               help = _VERBOSE_OPT_HELP)
-def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, verbose):
+def compile_script(yamlfile, platform, target, makejobs, nparallel, execute, verbose, force_compile):
     """ - Write the compile script """
     create_compile_script.compile_create(
-        yamlfile, platform, target, makejobs, nparallel, execute, verbose)
+        yamlfile, platform, target, makejobs, nparallel, execute, verbose, force_compile)
 
 @make_cli.command('dockerfile')
 @click.option("-y",

fre/make/gfdlfremake/buildBaremetal.py

@@ -6,28 +6,67 @@
 
 import subprocess
 import os
+from pathlib import Path
 
+### TODO run as a batch job on the login cluster
 def fremake_parallel(fremakeBuildList):
     """
-    Brief: Called for parallel execution purposes.  Runs the builds.
-    Param:
-        - fremakeBuildList : fremakeBuild object list passes by pool.map
+    Called for parallel execution purposes.  Runs the builds.
+
+    :param fremakeBuildList: list of compile scripts to execute
+    :type fremakeBuildList: .................
     """
-    fremakeBuildList.run()
+    bldDir = Path(fremakeBuildList).parent
+
+    # Run compile script
+    p1 = subprocess.Popen(fremakeBuildList, stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
+
+    # Direct output to log file as well
+    p2 = subprocess.Popen(["tee",f"{bldDir}/log.compile"], stdin=p1.stdout)
+
+    # Allow process1 to receive SIGPIPE is process2 exits
+    p1.stdout.close()
+    p2.communicate()
+
+    # wait for process1 to finish before checking return code
+    p1.wait()
+    if p1.returncode != 0:
+        return {1: f"{bldDir}/log.compile"}
+    else:
+        return {0: f"{bldDir}/log.compile"} 
 
 class buildBaremetal():
     """
-    Brief: Creates the build script to compile the model
-    Param:
-        - self : The buildScript object
-        - exp  : The experiment name
-        - mkTemplatePath : The template used by mkmf to compile the model
-        - srcDir : The source directory
-        - bldDir : The build directory
+    Class holding routines that will create the build script to compile the model.
+
+    :ivar str exp: The experiment name
+    :ivar str mkTemplatePath: The template used by mkmf to compile the model
+    :ivar str srcDir: The source directory
+    :ivar str bldDir: The build directory
+    :ivar str target:
+    :ivar str env_setup:
+    :ivar str jobs:
     """
     def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,env_setup,jobs):
         """
         Initialize variables and set-up the compile script.
+
+        :param self:
+        :type self:
+        :param exp:
+        :type exp:
+        :param mkTemplatePath:
+        :type mkTemplatePath:
+        :param srcDir:
+        :type srcDir:
+        :param bldDir:
+        :type bldDir:
+        :param target:
+        :type target:
+        :param env_setup:
+        :type env_setup:
+        :param jobs:
+        :type jobs:
         """
         self.e = exp
         self.t = target.gettargetName()
@@ -57,10 +96,12 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,env_setup,jobs):
 
     def writeBuildComponents(self, c):
         """
-        Brief: Adds components to the build script
-        Param:
-            - self : The build script object
-            - c : Component from the compile yaml
+        Adds components to the build script
+
+        :param self: The build script object
+        :type self:
+        :param c: Component from the compile yaml
+        :type c:
         """
         # Shorthand for component
         comp = c["component"]
@@ -111,40 +152,14 @@ def writeBuildComponents(self, c):
 ##TODO: add targets input
     def writeScript(self):
         """
-        Brief: Finishes and writes the build script
-        Param:
-            - self : The buildScript object
+        Finishes and writes the build script
+
+        :param self: The buildScript object
+        :type self:
         """
         self.f.write(f"cd {self.bld}\n")
         self.f.write(f"{self.make}\n")
         self.f.close()
 
         # Make compile script executable
         os.chmod(self.bld+"/compile.sh", 0o744)
-
-## TODO run as a batch job on the login cluster
-    def run(self):
-        """
-        Brief: Run the build script
-        Param:
-            - self : The dockerfile object
-        """
-        command = [self.bld+"/compile.sh"]
-
-        # Run compile script
-        p1 = subprocess.Popen(command, stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
-
-        # Direct output to log file as well
-        p2 = subprocess.Popen(["tee",self.bld+"/log.compile"], stdin=p1.stdout)
-
-        # Allow process1 to receive SIGPIPE is process2 exits
-        p1.stdout.close()
-        p2.communicate()
-
-        # wait for process1 to finish before checking return code
-        p1.wait()
-        if p1.returncode != 0:
-            print(f"\nThere was an error running {self.bld}/compile.sh")
-            print(f"Check the log file: {self.bld}/log.compile")
-        else:
-            print(f"\nSuccessful run of {self.bld}/compile.sh")

fre/make/gfdlfremake/yamlfre.py

@@ -5,6 +5,9 @@
 from jsonschema import validate, ValidationError, SchemaError
 from . import platformfre
 
+import logging
+fre_logger = logging.getLogger(__name__)
+
 def parseCompile(fname,v):
     """
     Brief: Open the yaml file and parse as fremakeYaml
@@ -52,22 +55,19 @@ def __init__(self,compileinfo):
         try:
             self.yaml["src"]
         except:
-            print("You must set a src to specify the sources in modelRoot/"+self.yaml["experiment"]+"\n")
-            raise
+            raise ValueError(f"You must set a src to specify the sources in modelRoot/{self.yaml['experiment']}")
         ## Loop through the src array
         for c in self.yaml['src']:
         ## Check for required component name
             try:
                 c['component']
             except:
-                print("You must set the 'component' name for each src component")
-                raise
+                raise ValueError("You must set the 'component' name for each src component")
             ## Check for required repo url
             try:
                 c['repo']
             except:
-                print("'repo' is missing from the component "+c['component']+" in "+self.yaml["experiment"]+"\n")
-                raise
+                raise ValueError(f"'repo' is missing from the component {c['component']} in {self.yaml['experiment']}")
             # Check for optional branch. Otherwise set it to blank
             try:
                 c['branch']
@@ -116,8 +116,7 @@ def getCompileYaml(self):
         try:
             self.yaml
         except:
-            print ("You must initialize the compile YAML object before you try to get the yaml \n")
-            raise
+            raise ValueError("You must initialize the compile YAML object before you try to get the yaml")
         return self.yaml
 
 class freyaml():
@@ -163,7 +162,7 @@ def __init__(self,combinedyaml,v):
         schema = json.loads(s)
 
         validate(instance=self.freyaml,schema=schema)
-        print("\nCOMBINED YAML VALID")
+        fre_logger.info(" *** COMBINED YAML VALID ***")
 
     def getCompileYaml(self):
         """