Upgraded to use Python3

esgprep/checkvocab/context.py

@@ -6,7 +6,7 @@
 
 """
 
-from constants import *
+from .constants import *
 from esgprep.utils.collectors import PathCollector, DatasetCollector, Collector
 from esgprep.utils.context import MultiprocessingContext
 from esgprep.utils.custom_print import *

esgprep/checkvocab/main.py

@@ -15,8 +15,8 @@
 from fuzzywuzzy.fuzz import partial_ratio
 from fuzzywuzzy.process import extractOne
 
-from constants import *
-from context import ProcessingContext
+from .constants import *
+from .context import ProcessingContext
 from esgprep.utils.custom_print import *
 from esgprep.utils.misc import ProcessContext, ncopen
 

esgprep/drs/context.py

@@ -8,8 +8,8 @@
 
 from uuid import uuid4 as uuid
 
-from constants import *
-from custom_exceptions import *
+from .constants import *
+from .custom_exceptions import *
 from esgprep.utils.collectors import Collector
 from esgprep.utils.context import MultiprocessingContext
 from esgprep.utils.custom_print import *

esgprep/drs/handler.py

@@ -18,8 +18,8 @@
 from treelib import Tree
 from treelib.tree import DuplicatedNodeIdError
 
-from constants import *
-from custom_exceptions import *
+from .constants import *
+from .custom_exceptions import *
 from esgprep.utils.custom_print import *
 from esgprep.utils.misc import ncopen
 

esgprep/drs/main.py

@@ -10,12 +10,12 @@
 import traceback
 from multiprocessing import Pool
 
-from constants import *
-from context import ProcessingContext
-from custom_exceptions import *
+from .constants import *
+from .context import ProcessingContext
+from .custom_exceptions import *
 from esgprep.utils.custom_print import *
 from esgprep.utils.misc import load, store, evaluate, ProcessContext, get_tracking_id, get_checksum
-from handler import File, DRSPath, DRSTree
+from .handler import File, DRSPath, DRSTree
 
 
 def process(source):

esgprep/esgcheckvocab.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 """
@@ -10,8 +10,8 @@
 
 from esgprep.checkvocab.main import run
 from esgprep.utils.help import *
-from utils.constants import *
-from utils.parser import *
+from .utils.constants import *
+from .utils.parser import *
 
 __version__ = 'from esgprep v{} {}'.format(VERSION, VERSION_DATE)
 

esgprep/esgdrs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 """
@@ -11,8 +11,8 @@
 
 from esgprep.drs.main import run
 from esgprep.utils.help import *
-from utils.constants import *
-from utils.parser import *
+from .utils.constants import *
+from .utils.parser import *
 
 __version__ = 'from esgprep v{} {}'.format(VERSION, VERSION_DATE)
 

esgprep/esgfetchini.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 """
@@ -8,9 +8,9 @@
 """
 
 from esgprep.fetchini.main import run
-from utils.constants import *
-from utils.help import *
-from utils.parser import *
+from .utils.constants import *
+from .utils.help import *
+from .utils.parser import *
 
 __version__ = 'from esgprep v{} {}'.format(VERSION, VERSION_DATE)
 

esgprep/esgfetchtables.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 """
@@ -8,9 +8,9 @@
 """
 
 from esgprep.fetchtables.main import run
-from utils.constants import *
-from utils.help import *
-from utils.parser import *
+from .utils.constants import *
+from .utils.help import *
+from .utils.parser import *
 
 __version__ = 'from esgprep v{} {}'.format(VERSION, VERSION_DATE)
 

esgprep/esgmapfile.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 """
@@ -11,8 +11,8 @@
 
 from esgprep.mapfile.main import run
 from esgprep.utils.help import *
-from utils.constants import *
-from utils.parser import *
+from .utils.constants import *
+from .utils.parser import *
 
 __version__ = 'from esgprep v{} {}'.format(VERSION, VERSION_DATE)
 

esgprep/fetchini/context.py

@@ -6,7 +6,7 @@
 
 """
 
-from constants import *
+from .constants import *
 from esgprep.utils.constants import GITHUB_API_PARAMETER
 from esgprep.utils.context import GitHubBaseContext
 from esgprep.utils.github import *

esgprep/fetchtables/context.py

@@ -6,7 +6,7 @@
 
 """
 
-from constants import *
+from .constants import *
 from esgprep.utils.collectors import FilterCollection
 from esgprep.utils.context import GitHubBaseContext
 from esgprep.utils.custom_print import *

esgprep/fetchtables/main.py

@@ -8,8 +8,8 @@
 
 import traceback
 
-from constants import *
-from context import ProcessingContext
+from .constants import *
+from .context import ProcessingContext
 from esgprep.utils.constants import GITHUB_API_PARAMETER
 from esgprep.utils.github import *
 

esgprep/mapfile/context.py

@@ -8,7 +8,7 @@
 
 import fnmatch
 
-from constants import *
+from .constants import *
 from esgprep.utils.collectors import VersionedPathCollector, DatasetCollector
 from esgprep.utils.context import MultiprocessingContext
 from esgprep.utils.custom_print import *

esgprep/mapfile/handler.py

@@ -12,7 +12,7 @@
 from ESGConfigParser import interpolate
 from ESGConfigParser.custom_exceptions import ExpressionNotMatch, NoConfigOption, MissingPatternKey
 
-from constants import *
+from .constants import *
 from esgprep.utils.custom_exceptions import *
 
 

esgprep/mapfile/main.py

@@ -13,13 +13,13 @@
 from ESGConfigParser import interpolate, MissingPatternKey, BadInterpolation, InterpolationDepthError
 from lockfile import LockFile
 
-from constants import *
-from context import ProcessingContext
-from custom_exceptions import *
+from .constants import *
+from .context import ProcessingContext
+from .custom_exceptions import *
 from esgprep.utils.custom_print import *
 from esgprep.utils.misc import evaluate, remove, get_checksum, ProcessContext
 from esgprep.utils.output_control import OutputControl
-from handler import File, Dataset
+from .handler import File, Dataset
 
 
 def get_output_mapfile(outdir, attributes, mapfile_name, dataset_id, dataset_version, mapfile_drs=None, basename=False):
@@ -221,7 +221,8 @@ def initializer(keys, values):
     """
     assert len(keys) == len(values)
     global pctx
-    pctx = ProcessContext({key: values[i] for i, key in enumerate(keys)})
+    v = list(values)
+    pctx = ProcessContext({key: v[i] for i, key in enumerate(keys)})
 
 
 def run(args):
@@ -266,11 +267,13 @@ def run(args):
         # Flush buffer
         Print.flush()
         # Get number of files scanned (excluding errors/skipped files)
-        ctx.scan_data = len(filter(None, results))
+        l = list(filter(None, results))
+        ctx.scan_data = len(l)
         # Get number of scan errors
         ctx.scan_errors = results.count(None)
         # Get number of generated mapfiles
-        ctx.nbmap = len(filter(None, set(results)))
+        l = list(filter(None, set(results)))
+        ctx.nbmap = len(l)
         # Evaluates the scan results to finalize mapfiles writing
         if evaluate(results):
             for mapfile in filter(None, set(results)):
@@ -280,7 +283,7 @@ def run(args):
                     result = remove(WORKING_EXTENSION, mapfile)
                     if quiet:
                         output_control.stdout_on()
-                        print result
+                        print (result)
                         output_control.stdout_off()
                     else:
                         Print.result(result)

esgprep/utils/collectors.py

@@ -227,7 +227,7 @@ class FilterCollection(object):
     indicating to match (i.e., include) or non-match (i.e., exclude) the corresponding expression.
 
     """
-    FILTER_TYPES = (str, re._pattern_type)
+    FILTER_TYPES = (str, re.Pattern)
 
     def __init__(self):
         self.filters = dict()

esgprep/utils/constants.py

@@ -12,7 +12,7 @@
 VERSION = '2.9.7'  # remember to change VERSION_DATE when updating
 
 # Date
-VERSION_DATE = datetime(year=2019, month=04, day=15).strftime("%Y-%d-%m")
+VERSION_DATE = datetime(year=2019, month=4, day=15).strftime("%Y-%d-%m")
 
 # Shell colors map
 SHELL_COLORS = {'red': 1,

esgprep/utils/context.py

@@ -127,7 +127,7 @@ def __init__(self, args):
             self.manager = SyncManager()
             self.manager.start()
             self.progress = self.manager.Value('i', 0)
-            Print.BUFFER = self.manager.Value(c_char_p, '')
+            Print.BUFFER = self.manager.Value(c_char_p, b'')
         else:
             self.progress = Value('i', 0)
         # Stdout lock
@@ -200,8 +200,8 @@ def get_checksum_type(self):
             checksum_type = _cfg.get_options_from_table('checksum')[0][1].lower()
         else:  # Use SHA256 as default because esg.ini not mandatory in configuration directory
             checksum_type = 'sha256'
-        if checksum_type not in checksum_types:
-            raise InvalidChecksumType(checksum_type)
+        #if checksum_type not in checksum_types:
+        #    raise InvalidChecksumType(checksum_type)
         _cfg.reset()
         return checksum_type
 

esgprep/utils/custom_exceptions.py

@@ -7,7 +7,7 @@
 """
 
 from datetime import datetime
-from hashlib import algorithms as checksum_types
+import hashlib
 
 
 ###############################
@@ -70,7 +70,7 @@ class InvalidChecksumType(Exception):
     def __init__(self, client):
         self.msg = "Checksum type not supported or invalid."
         self.msg += "\n<checksum type: '{}'>".format(client)
-        self.msg += "\n<allowed algorithms: '{}'>".format(checksum_types)
+        self.msg += "\n<allowed algorithms: '{}'>".format(algorithms)
         super(self.__class__, self).__init__(self.msg)
 
 

esgprep/utils/custom_print.py

@@ -12,8 +12,8 @@
 from ctypes import c_char_p
 from multiprocessing import Value
 
-from constants import SHELL_COLORS
-from custom_exceptions import *
+from .constants import SHELL_COLORS
+from .custom_exceptions import *
 
 _colors_enabled = [sys.stdout.isatty()]
 
@@ -181,7 +181,7 @@ class Print(object):
     DEBUG = False
     CMD = None
     LOG_TO_STDOUT = False
-    BUFFER = Value(c_char_p, '')
+    BUFFER = Value(c_char_p, b'')
     LOGFILE = None
     CARRIAGE_RETURNED = True
 

esgprep/utils/help.py

@@ -6,7 +6,7 @@
 
 """
 
-from custom_print import *
+from .custom_print import *
 
 # Help
 TITLE = COLOR('yellow').bold(""".___ ___ ___ ___ ___ ___ ___.

esgprep/utils/misc.py

@@ -12,7 +12,7 @@
 
 from netCDF4 import Dataset
 
-from custom_print import *
+from .custom_print import *
 from esgprep.drs.constants import PID_PREFIXES
 
 

requirements.txt

@@ -1,8 +1,6 @@
-lockfile==0.12.2
-cftime==1.0.3.4
-esgconfigparser==0.1.17
-requests==2.20.0
-fuzzywuzzy==0.16.0
-netCDF4==1.4.0
-hurry.filesize==0.9
-treelib==1.4.0
+lockfile
+cftime
+requests
+fuzzywuzzy
+hurry.filesize
+treelib

setup.py

@@ -65,15 +65,15 @@ def run(self):
           url='https://github.com/ESGF/esgf-prepare',
           packages=find_packages(),
           include_package_data=True,
-          python_requires='>=2.7, <3.0',
-          install_requires=['lockfile==0.12.2',
-                            'cftime==1.0.3.4'
-                            'esgconfigparser==0.1.17',
-                            'requests==2.20.0',
-                            'fuzzywuzzy==0.16.0',
-                            'netCDF4==1.4.0',
-                            'hurry.filesize==0.9',
-                            'treelib==1.4.0'],
+          python_requires='>=3.0, <4.0',
+          install_requires=['lockfile',
+                            'cftime',
+                            'esgconfigparser',
+                            'requests',
+                            'fuzzywuzzy',
+                            'netCDF4',
+                            'hurry.filesize',
+                            'treelib'],
           platforms=['Unix'],
           zip_safe=False,
           entry_points={'console_scripts': ['esgmapfile=esgprep.esgmapfile:main',