Added support for python 3 for DIY DSC (#789)

Summary of changes included as a part of the PR:

Porting changes (Fixes) to run all scripts on python 3
Updated DSC default resources offered under nx to run on python 3.
Added additional logging to all python scripts.
Fixes for bugs that were introduced as a part of DROP OMI Depenedency.
Description of the changes:
As a part of porting the code we had decided to keep the code for python3 in a seperate forlder.(Most of these changes were done as a part of OMSConfig Py3 support PR). The updated folder for python scripts is as follows:

python2 : /opt/microsoft/dsc/Scripts
python3: /opt/microsoft/dsc/Scripts/python3
The python version of the machine will be determined by checking the existance of python2 and python3 command. In this version python2 takes precidence if both python 2 and 3 are present. (This is done to maintain compabality for older distros.)

Instead of just logging success and failure a lot of debug and detailed logs ave been added. Going forward DSC will have two log files:

/var/opt/omi/log/dsc.log
/var/opt/omi/log/dscdetailed.log
A lot of issues were discovered for the master branch, as the last release was done years back. I have tried to fix most of them. The issue were as a part of the DROP OMI DEPENDECY pr, which was only intended for omsconfig and ended up breaking DIY DSC.

Testing:
Manually tested for sanity and happy path secnarios on both python 2 and 3.
Distro testing is in progress using the test suite.
Bug bas planned for this week.


* Initial Changes to run locally

* Update InstallModule.py

* Updated helper scripts to account for new paths

* Updated providers to get invoked with python3 instead of python

* Fix FindFirstFile Failed bug that was introduced in 2019

* Fixed path to omicli for scripts

* Fixed the bug causing test to fail, it was introduced by a 2019 change DROP OMI DEPENDENCY

* Update CAEngine.c

* Added logs to python scripts

* Updated logging to capture python major version

* Added logs around compare in nxFile

* Removed logging from import module to get rid of circular dependency

* Removed logging from RegenerateInit to remove circular dependency

* Update RegenerateInitFiles.py

* Fixed comparison in case of python 3 for nxFile

* Updated imports for loggig to work

* Using absolute path instead of relative paths for DSCLogger

* Update Makefile to respect version provided in Build Repo

* Updated paths

* Updated Register.py

* Updated to remove warnings thrown for 'is'

* Update if such that we check instance details before reginstance

* Additioal logs to debug a failure in get configuration will revert the ones not required

* Removed additional logging (These were just for debugging and not required to be released)

* Removed additional logs

* Updating content to string as OMI is not able to handle bytes

* Resolving sekhar's comments

Author: Jinesh Shah

LCM/dsc/engine/ConsistencyInvoker/ConsistencyInvoker.c

@@ -31,20 +31,15 @@ int main(int argc, char *argv[])
 
     char* dscScriptPath = malloc(strlen(DSC_SCRIPT_PATH) + 1);
     dscScriptPath = strcpy(dscScriptPath, DSC_SCRIPT_PATH);
-    
-    //if oms config check to be added
-    if(strstr(dscScriptPath, "omsconfig")!= NULL)
-    {
-        pythonCommand = getPythonProvider();
-
-        if(strcmp(pythonCommand, PYTHON3_COMMAND) == 0)
-        {
-            dscScriptPath = realloc(dscScriptPath, strlen(dscScriptPath) + strlen("/python3") + 1 );
-            dscScriptPath = strcat(dscScriptPath, "/python3");
-        }
-    }
 
+    pythonCommand = getPythonProvider();
 
+    if(strcmp(pythonCommand, PYTHON3_COMMAND) == 0)
+    {
+        dscScriptPath = realloc(dscScriptPath, strlen(dscScriptPath) + strlen("/python3") + 1 );
+        dscScriptPath = strcat(dscScriptPath, "/python3");
+    }
+    
     int fullCommandLength = strlen(pythonCommand) + 1 + strlen(dscScriptPath) + 1 + strlen(PYTHON_SCRIPT_NAME) + 1;
     char fullCommand[fullCommandLength];
 
@@ -59,7 +54,6 @@ int main(int argc, char *argv[])
 }
 
 
-// I may need to move this method in some file which is accessible to all other files in the project.
 char* getPythonProvider()
 {
     int buffer_length = 128;

LCM/dsc/engine/ModuleLoader/ModuleLibrary/ModuleHandler.c

@@ -469,7 +469,6 @@ MI_Result GetModuleLoader( _In_ MI_Application *miApp,
 #if defined(BUILD_OMS)
     if (g_DscHost == MI_FALSE)
     {
-#endif
         // Get the Registration information from shared objects
         r = GetRegistrationInstanceFromSharedObjects(NULL, miApp, de, options, strictOptions, &miClassArray, &miInstanceArray, extendedError);
         if( r != MI_RESULT_OK)
@@ -482,6 +481,7 @@ MI_Result GetModuleLoader( _In_ MI_Application *miApp,
             CleanUpInstanceCache(&miInstanceArray);
             return r;
         }
+#endif
 
         // No need to do this after removing OMI since we will discover resources directly from shared objects (except for DIY DSC)
         /*Perform registration against schema validation*/

LCM/dsc/engine/ca/CAInfrastructure/CAEngine.c

@@ -1286,10 +1286,11 @@ MI_Result MoveToDesiredState(_In_ ProviderCallbackContext *provContext,
     if (
 #if defined(BUILD_OMS)
         g_DscHost == MI_FALSE ||
+        Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
 #else
+        Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
         Tcscasecmp(regInstance->classDecl->name, BASE_REGISTRATION_WMIV2PROVIDER) == 0 ||
 #endif
-        Tcscasecmp(instance->classDecl->name, METACONFIG_CLASSNAME) == 0 || // put special cases to wmiv2 code
         Tcscasecmp(MSFT_LOGRESOURCENAME, instance->classDecl->name) == 0
         )
     {
@@ -1910,15 +1911,16 @@ MI_Result GetGetMethodResult(_In_ MI_Operation *operation,
     *extendedError = NULL;  // Explicitly set *extendedError to NULL as _Outptr_ requires setting this at least once.
 
     *outputInstance = NULL;
-
     /*Get the operation result*/
     r = MI_Operation_GetInstance(operation, &outInstance, &moreResults, &result, &errorMessage, &completionDetails);
     if( result != MI_RESULT_OK)
     {
+        DSC_LOG_INFO("MI_Operation_GetInstance inside GetGetMethodResult failed result not ok");
         r = result;
     }
     if( r != MI_RESULT_OK)
     {
+        DSC_LOG_INFO("MI_Operation_GetInstance inside GetGetMethodResult failed r not ok");
         if( completionDetails != NULL)
         {
             innerR = DSC_MI_Instance_Clone( completionDetails, extendedError);
@@ -1927,6 +1929,7 @@ MI_Result GetGetMethodResult(_In_ MI_Operation *operation,
         {
             r = GetCimMIError(r, extendedError,ID_CAINFRA_GETINSTANCE_FAILED);
         }
+
         return r;
     }
 
@@ -2021,18 +2024,21 @@ MI_Result Get_WMIv2Provider(_In_ ProviderCallbackContext *provContext,
             return GetCimMIError(r, extendedError,ID_CAINFRA_GET_NEWAPPLICATIONINSTANCE_FAILED);
         }
         value.instance = instance;
+        
         r = DSC_MI_Instance_AddElement(params, OMI_BaseResource_Method_InputResource, &value, MI_INSTANCE, 0 );
         if( r != MI_RESULT_OK)
         {
             MI_Instance_Delete(params);
             return GetCimMIError(r, extendedError,ID_CAINFRA_GET_ADDELEM_FAILED);
         }
+        
         r = MI_Application_NewOperationOptions(miApp, MI_FALSE, &sessionOptions);
         if( r != MI_RESULT_OK )
         {
             return GetCimMIError(r, extendedError,ID_CAINFRA_GET_NEWOPERATIONOPTIONS_FAILED);
         }
         valueOperationOptions.string=g_ConfigurationDetails.jobGuidString;
+        
         r =MI_OperationOptions_SetCustomOption(&sessionOptions,DSC_JOBIDSTRING,MI_STRING,&valueOperationOptions,MI_FALSE);
         if( r != MI_RESULT_OK)
         {
@@ -2041,17 +2047,16 @@ MI_Result Get_WMIv2Provider(_In_ ProviderCallbackContext *provContext,
         }
 
         /* Perform Get*/
-
         MI_Session_Invoke(miSession, 0, &sessionOptions, provNamespace,
                              instance->classDecl->name, OMI_BaseResource_GetMethodName,
                              NULL, params, &callbacks,&operation);
-
         r = GetGetMethodResult(&operation, outputInstance , extendedError);
         MI_Instance_Delete(params);
         MI_OperationOptions_Delete(&sessionOptions);
         MI_Operation_Close(&operation);
         if( r != MI_RESULT_OK)
         {
+            DSC_LOG_INFO("GetGetMethodResult r is not ok");
             return r;
         }
     }

LCM/scripts/GetDscConfiguration.py

@@ -8,24 +8,31 @@
 import os
 import os.path
 from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
-from imp                import load_source
-from os.path            import dirname, isfile, join, realpath
-from fcntl              import flock, LOCK_EX, LOCK_UN, LOCK_NB
+from imp            import load_source
+from os.path        import dirname, isfile, join, realpath
+from fcntl          import flock, LOCK_EX, LOCK_UN, LOCK_NB
+from sys            import argv
+
 
 pathToCurrentScript = realpath(__file__)
 pathToCommonScriptsFolder = dirname(pathToCurrentScript)
 
+DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
+nxDSCLog = load_source('nxDSCLog', DSCLogPath)
+LG = nxDSCLog.DSCLog
+
 helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
 helperlib = load_source('helperlib', helperLibPath)
 
-omi_bindir = "<CONFIG_BINDIR>"
-omicli_path = omi_bindir + "/omicli"
+omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
 dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
 dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
 dsc_host_output_path = join(dsc_host_base_path, 'output')
 dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
 dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')
 
+LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
+
 if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
     write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))
 
@@ -93,4 +100,5 @@
 print(stdout)
 print(stderr)
 
+LG().Log("DEBUG", "End of script logic for " +  argv[0] + " runing with python " + str(sys.version_info.major))
 

LCM/scripts/GetDscLocalConfigurationManager.py

@@ -8,24 +8,30 @@
 import os
 import os.path
 from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
-from imp                import load_source
-from os.path            import dirname, isfile, join, realpath
-from fcntl              import flock, LOCK_EX, LOCK_UN, LOCK_NB
+from imp            import load_source
+from os.path        import dirname, isfile, join, realpath
+from fcntl          import flock, LOCK_EX, LOCK_UN, LOCK_NB
+from sys            import argv
 
 pathToCurrentScript = realpath(__file__)
 pathToCommonScriptsFolder = dirname(pathToCurrentScript)
 
+DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
+nxDSCLog = load_source('nxDSCLog', DSCLogPath)
+LG = nxDSCLog.DSCLog
+
 helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
 helperlib = load_source('helperlib', helperLibPath)
 
-omi_bindir = "<CONFIG_BINDIR>"
-omicli_path = omi_bindir + "/omicli"
+omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
 dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
 dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
 dsc_host_output_path = join(dsc_host_base_path, 'output')
 dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
 dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')
 
+LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
+
 if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
     write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))
 
@@ -93,4 +99,5 @@
 print(stdout)
 print(stderr)
 
+LG().Log("DEBUG", "End of script logic for " +  argv[0] + " runing with python " + str(sys.version_info.major))
 

LCM/scripts/InstallModule.py

@@ -6,7 +6,7 @@
 import subprocess
 import sys
 import platform
-from os.path import basename, dirname, join, realpath, split
+from os.path    import basename, dirname, join, realpath, split
 
 pathToCurrentScript = realpath(__file__)
 pathToCommonScriptsFolder = dirname(pathToCurrentScript)
@@ -357,4 +357,4 @@ def main(args):
     regenerateDscPythonScriptInitFiles()
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    main(sys.argv[1:])

LCM/scripts/PerformRequiredConfigurationChecks.py

@@ -1,8 +1,9 @@
 #!/usr/bin/python
+import sys
 from imp        import load_source
 from os.path    import dirname, join, realpath, isfile
 from subprocess import PIPE, Popen
-from sys        import exc_info, exit, version_info
+from sys        import exc_info, exit, version_info, argv
 from traceback  import format_exc
 from fcntl      import flock, LOCK_EX, LOCK_UN, LOCK_NB
 from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
@@ -11,6 +12,10 @@
 pathToCurrentScript = realpath(__file__)
 pathToCommonScriptsFolder = dirname(pathToCurrentScript)
 
+DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
+nxDSCLog = load_source('nxDSCLog', DSCLogPath)
+LG = nxDSCLog.DSCLog
+
 helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
 helperlib = load_source('helperlib', helperLibPath)
 
@@ -111,4 +116,6 @@ def run_perform_required_configuration_checks():
     print(stdout)
 
 if __name__ == "__main__":
+    LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
     main()
+    LG().Log("DEBUG", "End of Main method for " +  argv[0] + " runing with python " + str(sys.version_info.major))

LCM/scripts/RegenerateInitFiles.py

@@ -8,6 +8,7 @@
              omi_libdir + "/2.6x-2.7x/Scripts",
              omi_libdir + "/3.x/Scripts"]
 
+
 for current_dir in script_dirs:
     out_init = "__all__="
     py_files = glob.glob(current_dir + "/*.py")
@@ -24,5 +25,4 @@
         py_files_basename.append(current_basename[:-3])
 
     out_init = out_init + str(py_files_basename)
-    open(current_dir + "/__init__.py", "w").write(out_init)
-        
+    open(current_dir + "/__init__.py", "w").write(out_init)

LCM/scripts/Register.py

@@ -4,6 +4,9 @@
 import os.path
 import tempfile
 import shutil
+from sys                import argv
+from imp                import load_source
+from os.path import dirname, join, realpath
 
 def usage():
    print("""Usage: Register.py [OPTIONS]
@@ -18,6 +21,16 @@ def usage():
  --Help
 """)
 
+
+pathToCurrentScript = realpath(__file__)
+pathToCommonScriptsFolder = dirname(pathToCurrentScript)
+
+DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
+nxDSCLog = load_source('nxDSCLog', DSCLogPath)
+LG = nxDSCLog.DSCLog
+
+LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
+
 # Apply a DSC meta configuration based on a template
 Variables = dict()
 
@@ -217,3 +230,5 @@ def usage():
 os.system("<DSC_SCRIPT_PATH>/SetDscLocalConfigurationManager.py -configurationmof " + meta_path)
 
 shutil.rmtree(tempdir)
+
+LG().Log("DEBUG", "End of script logic for " +  argv[0] + " runing with python " + str(sys.version_info.major))

LCM/scripts/RemoveModule.py

@@ -6,9 +6,16 @@
 import platform
 import imp
 from os.path import dirname, join, realpath
+from sys                  import argv
+from imp        import load_source
 
 pathToCurrentScript = realpath(__file__)
 pathToCommonScriptsFolder = dirname(pathToCurrentScript)
+
+DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
+nxDSCLog = load_source('nxDSCLog', DSCLogPath)
+LG = nxDSCLog.DSCLog
+
 helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
 helperlib = imp.load_source('helperlib', helperLibPath)
 fullPathDSCLogger = os.path.join(pathToCommonScriptsFolder, 'nxDSCLog.py')
@@ -196,4 +203,6 @@ def main(args):
     shutil.rmtree(modulePath)
 
 if __name__ == "__main__":
+    LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
     main(sys.argv[1:])
+    LG().Log("DEBUG", "End of Main method for " +  argv[0] + " runing with python " + str(sys.version_info.major))

LCM/scripts/RestoreConfiguration.py

@@ -7,21 +7,28 @@
 from fcntl                import flock, LOCK_EX, LOCK_UN, LOCK_NB
 from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
 from time                 import sleep
+from sys                  import argv
+
 
 pathToCurrentScript = realpath(__file__)
 pathToCommonScriptsFolder = dirname(pathToCurrentScript)
 
+DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
+nxDSCLog = load_source('nxDSCLog', DSCLogPath)
+LG = nxDSCLog.DSCLog
+
 helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
 helperlib = load_source('helperlib', helperLibPath)
 
-omi_bindir = "<CONFIG_BINDIR>"
-omicli_path = omi_bindir + "/omicli"
+omicli_path = join(helperlib.CONFIG_BINDIR, 'omicli')
 dsc_host_base_path = helperlib.DSC_HOST_BASE_PATH
 dsc_host_path = join(dsc_host_base_path, 'bin/dsc_host')
 dsc_host_output_path = join(dsc_host_base_path, 'output')
 dsc_host_lock_path = join(dsc_host_base_path, 'dsc_host_lock')
 dsc_host_switch_path = join(dsc_host_base_path, 'dsc_host_ready')
 
+LG().Log("DEBUG", "Starting script logic for " + argv[0]+ " runing with python " + str(sys.version_info.major))
+
 if ("omsconfig" in helperlib.DSC_SCRIPT_PATH):
     write_omsconfig_host_switch_event(pathToCurrentScript, isfile(dsc_host_switch_path))
 
@@ -86,3 +93,5 @@
 
 print(stdout)
 print(stderr)
+
+LG().Log("DEBUG", "End of script logic for " +  argv[0] + " runing with python " + str(sys.version_info.major))

LCM/scripts/SetDscLocalConfigurationManager.py

@@ -8,10 +8,16 @@
 from OmsConfigHostHelpers import write_omsconfig_host_telemetry, write_omsconfig_host_switch_event, write_omsconfig_host_log, stop_old_host_instances
 from time                 import sleep
 import signal
+import sys
+
 
 pathToCurrentScript = realpath(__file__)
 pathToCommonScriptsFolder = dirname(pathToCurrentScript)
 
+DSCLogPath = join(pathToCommonScriptsFolder, 'nxDSCLog.py')
+nxDSCLog = load_source('nxDSCLog', DSCLogPath)
+LG = nxDSCLog.DSCLog
+
 helperLibPath = join(pathToCommonScriptsFolder, 'helperlib.py')
 helperlib = load_source('helperlib', helperLibPath)
 
@@ -170,4 +176,6 @@ def signal_handler(signalNumber, frame):
 if __name__ == "__main__":
     # register the SIGTERM handler
     signal.signal(signal.SIGTERM, signal_handler)
+    LG().Log("DEBUG", "Starting Main method for " + argv[0] + " runing with python " + str(sys.version_info.major))
     main(argv)
+    LG().Log("DEBUG", "End of Main method for " +  argv[0] + " runing with python " + str(sys.version_info.major))