New master

.github/workflows/basic.yml

@@ -24,7 +24,6 @@ jobs:
       fail-fast: false
       matrix:
         python: 
-          - 2.7.18
           - 3.6.15
           - 3.9.17
 
@@ -55,7 +54,6 @@ jobs:
       fail-fast: false
       matrix:
         python: 
-          - 2.7.18
           - 3.6.15
           - 3.9.17
 
@@ -82,7 +80,6 @@ jobs:
       fail-fast: false
       matrix:
         python: 
-          - 2.7.18
           - 3.6.15
           - 3.9.17
 

.github/workflows/integration.yml

@@ -43,7 +43,7 @@ jobs:
           pip install DIRAC
           pilot_proxy_file=$(mktemp)
           dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
-          dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file
+          dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file
           cd ${GITHUB_WORKSPACE}/Pilot
           cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -59,17 +59,14 @@ jobs:
           pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
           X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_branch }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug
 
-  release_prod_pre-cvmfs_CEs:
+  release_cvmfs_CEs:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
         pilot_schema:
           - pilot_oldSchema.json
           - pilot_newSchema.json
-        dirac_version:
-          - prod
-          - old
         ce:
           - jenkins.cern.ch
           - jenkins-singularity.cern.ch
@@ -98,15 +95,11 @@ jobs:
           echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
           chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
           chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
-          if [ "${{ matrix.dirac_version }}" == "prod" ]; then
-            version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
-          else
-            version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1)
-          fi
+          version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
           source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
           pilot_proxy_file=$(mktemp)
           dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
-          dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
+          dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
           cd ${GITHUB_WORKSPACE}/Pilot
           cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -156,7 +149,7 @@ jobs:
           source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
           pilot_proxy_file=$(mktemp)
           dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
-          dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
+          dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
           cd ${GITHUB_WORKSPACE}/Pilot
           cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -172,18 +165,14 @@ jobs:
           pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
           X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
 
-  release_prod_pre-cvmfs_matching:
+  release_cvmfs_matching:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
         VO:
           - dteam
           - gridpp
-        dirac_version:
-          - prod
-          - old
-
     steps:
       - uses: actions/checkout@v4
       - uses: cvmfs-contrib/github-action-cvmfs@v3
@@ -206,15 +195,11 @@ jobs:
           chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
           chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
           cd ${GITHUB_WORKSPACE}/Pilot
-          if [ "${{ matrix.dirac_version }}" == "prod" ]; then
-            version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
-          else
-            version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1)
-          fi
+          version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
           source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
           pilot_proxy_file=$(mktemp)
           dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
-          dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
+          dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
           cd ${GITHUB_WORKSPACE}/Pilot
           cp ../tests/CI/pilot_newSchema.json pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -267,7 +252,7 @@ jobs:
           source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
           pilot_proxy_file=$(mktemp)
           dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
-          dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
+          dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
           cd ${GITHUB_WORKSPACE}/Pilot
           cp ../tests/CI/pilot_newSchema.json pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -327,7 +312,7 @@ jobs:
           source /cvmfs/lhcb.cern.ch/lhcbdirac/${version}/Linux-x86_64/diracosrc
           pilot_proxy_file=$(mktemp)
           dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
-          X509_CERT_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/certificates" X509_VOMS_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir" DIRAC_VOMSES="/cvmfs/lhcb.cern.ch/etc/grid-security/vomses" dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
+          X509_CERT_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/certificates" X509_VOMS_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir" DIRAC_VOMSES="/cvmfs/lhcb.cern.ch/etc/grid-security/vomses" dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
           cd ${GITHUB_WORKSPACE}/Pilot
           export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot
           curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py
@@ -382,7 +367,7 @@ jobs:
           source /cvmfs/lhcb.cern.ch/lhcbdirac/${version}/Linux-x86_64/diracosrc
           pilot_proxy_file=$(mktemp)
           dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
-          dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
+          dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
           cd ${GITHUB_WORKSPACE}/Pilot
           curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py
           cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json

.pylintrc

@@ -18,7 +18,3 @@ dummy-variables=_
 disable=
     invalid-name,
     line-too-long, # would be nice to remove this one
-    consider-using-f-string, # python2/3 support
-    unspecified-encoding, # python2/3 support
-    super-with-arguments, # python2/3 support
-    redefined-builtin, # python2/3 support

Pilot/dirac-pilot.py

@@ -19,36 +19,19 @@
 But, as said, all the actions are actually configurable.
 """
 
-from __future__ import absolute_import, division, print_function
-
 import os
 import sys
 import time
+from io import StringIO
 
-############################
-# python 2 -> 3 "hacks"
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from io import StringIO
+from pilotTools import (
+    Logger,
+    PilotParams,
+    RemoteLogger,
+    getCommand,
+    pythonPathCheck,
+)
 
-try:
-    from Pilot.pilotTools import (
-        Logger,
-        PilotParams,
-        RemoteLogger,
-        getCommand,
-        pythonPathCheck,
-    )
-except ImportError:
-    from pilotTools import (
-        Logger,
-        PilotParams,
-        RemoteLogger,
-        getCommand,
-        pythonPathCheck,
-    )
 ############################
 
 if __name__ == "__main__":

Pilot/pilotCommands.py

@@ -17,8 +17,6 @@ def __init__(self, pilotParams):
 execution.
 """
 
-from __future__ import absolute_import, division, print_function
-
 import filecmp
 import os
 import platform
@@ -28,39 +26,18 @@ def __init__(self, pilotParams):
 import sys
 import time
 import traceback
-import subprocess
 from collections import Counter
+from http.client import HTTPSConnection
+from shlex import quote
+
+from pilotTools import (
+    CommandBase,
+    getSubmitterInfo,
+    retrieveUrlTimeout,
+    safe_listdir,
+    sendMessage,
+)
 
-############################
-# python 2 -> 3 "hacks"
-try:
-    # For Python 3.0 and later
-    from http.client import HTTPSConnection
-except ImportError:
-    # Fall back to Python 2
-    from httplib import HTTPSConnection
-
-try:
-    from shlex import quote
-except ImportError:
-    from pipes import quote
-
-try:
-    from Pilot.pilotTools import (
-        CommandBase,
-        getSubmitterInfo,
-        retrieveUrlTimeout,
-        safe_listdir,
-        sendMessage,
-    )
-except ImportError:
-    from pilotTools import (
-        CommandBase,
-        getSubmitterInfo,
-        retrieveUrlTimeout,
-        safe_listdir,
-        sendMessage,
-    )
 ############################
 
 
@@ -138,15 +115,19 @@ def execute(self):
         self.log.info("Host FQDN  = %s" % socket.getfqdn())
         self.log.info("WorkingDir = %s" % self.pp.workingDir)  # this could be different than rootPath
 
-        fileName = "/etc/redhat-release"
-        if os.path.exists(fileName):
-            with open(fileName, "r") as f:
-                self.log.info("RedHat Release = %s" % f.read().strip())
-
-        fileName = "/etc/lsb-release"
-        if os.path.isfile(fileName):
-            with open(fileName, "r") as f:
-                self.log.info("Linux release:\n%s" % f.read().strip())
+        for fileName in ["/etc/os-release", "/usr/lib/os-release"]:
+            if os.path.isfile(fileName):
+                try:
+                    with open(fileName, "r") as f:
+                        for line in f:
+                            line = line.strip()
+                            if line.startswith(("NAME=", "VERSION=", "PRETTY_NAME=")):
+                                self.log.info(
+                                    "OS Release = %s" % line.split("=", 1)[1].strip('"')
+                                )
+                    break
+                except (OSError, IOError):
+                    self.log.debug("Could not read %s" % fileName)
 
         fileName = "/proc/cpuinfo"
         if os.path.exists(fileName):
@@ -283,7 +264,7 @@ def _getPreinstalledEnvScript(self):
                 self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript
 
     def _localInstallDIRAC(self):
-        """Install python3 version of DIRAC client"""
+        """Install DIRAC client"""
 
         self.log.info("Installing DIRAC locally")
 
@@ -296,10 +277,7 @@ def _localInstallDIRAC(self):
 
         # 1. Get the DIRACOS installer name
         # curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh
-        try:
-            machine = os.uname().machine  # py3
-        except AttributeError:
-            machine = os.uname()[4]  # py2
+        machine = os.uname().machine
 
         installerName = "DIRACOS-Linux-%s.sh" % machine