Feature/compose file (#2)

* Translate yml to json

* Add support for service private environment variables, many fixes

* Update version numbers

* Fix container revert at naming

config.py

@@ -5,7 +5,7 @@
 
 # Config values
 # Version number
-servermanagerversion = "1.1.1"
+servermanagerversion = "1.2.0"
 
 # file paths
 servicepath = "/var/lib/servermanager/services/"

install.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
 # SchoolConnect Server-Installer
-# © 2019 Johannes Kreutz.
+# © 2019 - 2021 Johannes Kreutz.
 
-version='1.1.1'
+version='1.2.0'
 
 # Check for root rights
 if [[ $EUID > 0 ]]; then

modules/builderThread.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # SchoolConnect Server-Manager - image builder thread class
-# © 2019 Johannes Kreutz.
+# © 2019 - 2021 Johannes Kreutz.
 
 # Include dependencies
 import threading
@@ -36,6 +36,6 @@ def __buildImage(self, object):
 
     # Build a container with given image and description
     def __buildContainer(self, image, object):
-        containerObject = container.container(False, None)
+        containerObject = container.container(False, None, object["name"])
         containerObject.create(image, object["object"], object["volumeSource"])
         return containerObject

modules/container.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # SchoolConnect Server-Manager - container class
-# © 2019 Johannes Kreutz.
+# © 2019 - 2021 Johannes Kreutz.
 
 # Container status codes
 # 0: Clean init
@@ -17,18 +17,19 @@
 
 # Include modules
 import config
-import modules.envstore as envstore
+from modules.envstore import envman
 
 # Manager objects
-env = envstore.envman()
+env = envman()
 
 # Create docker connection
 client = docker.from_env()
 
 # Class definition
 class container:
-    def __init__(self, exists, id):
+    def __init__(self, exists, id, name):
         self.__status = 0
+        self.__localEnv = envman(name)
         if exists:
             try:
                 self.__container = client.containers.get(container_id=id)
@@ -145,7 +146,11 @@ def __createEnvironmentMap(self, object):
         environmentMap = {}
         if "environment" in object:
             for var in object["environment"]:
-                environmentMap[var] = env.getValue(var)
+                local = self.__localEnv.getValue(var)
+                if local is False:
+                    environmentMap[var] = env.getValue(var)
+                else:
+                    environmentMap[var] = local
         if object["name"] == "pc_admin":
             apitokenfile = open(config.configpath + config.apitokenfile, "r")
             environmentMap["APIKEY"] = apitokenfile.read()

modules/envparser.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+# SchoolConnect Server-Manager - env file parser (rewritten to use the comments above a key-value pair)
+# © 2021 Johannes Kreutz.
+
+# Class definition
+class envParser:
+    def __init__(self, path):
+        self.__path = path
+
+    # Return the environment object for the given file
+    def getEnvironment(self):
+        envList = []
+        with open(self.__path, "r") as f:
+            lastComment = ""
+            for line in f.readlines():
+                if line.startswith("#"):
+                    lastComment = line.strip()
+                else:
+                    if "=" in line:
+                        l = line.split("=")
+                        mutable = True if lastComment.endswith(" MU") or lastComment.endswith(" UM") or lastComment.endswith(" M") else False
+                        useValue = True if lastComment.endswith(" MU") or lastComment.endswith(" UM") or lastComment.endswith(" U") else False
+                        private = False if lastComment.endswith(" S") else True
+                        if lastComment.endswith(" MU") or lastComment.endswith(" UM"):
+                            lastComment = lastComment[:-2]
+                        if lastComment.endswith(" U") or lastComment.endswith(" M") or lastComment.endswith(" S"):
+                            lastComment = lastComment[:-1]
+                        if lastComment.startswith("#"):
+                            lastComment = lastComment[1:]
+                        lastComment = lastComment.strip()
+                        env = {
+                            "name": l[0].strip(),
+                            "description": lastComment,
+                            "mutable": mutable,
+                            "private": private,
+                        }
+                        if useValue:
+                            env["default"] = l[1].strip()
+                        envList.append(env)
+                        lastComment = ""
+                    elif len(line) <= 1:
+                        continue
+                    else:
+                        return None
+        return envList

modules/envstore.py

@@ -13,21 +13,28 @@
 
 # Class definition
 class envman:
-    def __init__(self):
-        if not os.path.exists(config.configpath + "env.json"):
-            self.__storage = {}
+    def __init__(self, service = None):
+        if service is None:
+            self.__path = config.configpath + "env.json"
+        else:
+            self.__path = config.servicepath + service + "/env.json"
+        self.__storage = {}
+        if not os.path.exists(self.__path):
             self.write()
-        self.updateLocalIp()
+        else:
+            self.load()
+        if service is None:
+            self.updateLocalIp()
 
     # Read stored environment variables and their descriptions from configuration files
     def load(self):
-        envfile = open(config.configpath + "env.json", "r")
+        envfile = open(self.__path, "r")
         self.__storage = json.loads(envfile.read())
         envfile.close()
 
     # Writes actual environment variables and their descriptions to the storage files
     def write(self):
-        envfile = open(config.configpath + "env.json", "w")
+        envfile = open(self.__path, "w")
         envfile.write(json.dumps(self.__storage, sort_keys=True, indent=4))
         envfile.close()
 

modules/image.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # SchoolConnect Server-Manager - image class
-# © 2019 Johannes Kreutz.
+# © 2019 - 2021 Johannes Kreutz.
 
 # Image status definitions
 # 0: Clean init
@@ -50,8 +50,10 @@ def create(self, object, wantedVersion):
         self.__wantedVersion = wantedVersion
         if "prebuilt" in self.__wantedVersion:
             return self.__pull(self.__wantedVersion["prebuilt"]["name"] + ":" + self.__wantedVersion["prebuilt"]["version"])
-        else:
+        elif "url" in self.__wantedVersion:
             return self.__build(self.__wantedVersion["url"])
+        else:
+            return self.__build(self.__wantedVersion["path"], False)
 
     # Pulls an image from the docker hub
     def __pull(self, name):
@@ -64,28 +66,32 @@ def __pull(self, name):
             return False
 
     # Builds an image from a given url
-    def __build(self, url):
-        if not os.path.exists(config.servicepath + "buildcache"):
-            os.makedirs(config.servicepath + "buildcache")
-        randomString = ess.essentials.randomString(10)
-        fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString)
-        fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz")
-        os.makedirs(config.servicepath + "buildcache/" + self.__name + "_" + randomString)
-        urllib.request.urlretrieve(url, config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz")
-        tar = Popen(["/bin/tar", "-zxf", config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz", "--directory", config.servicepath + "buildcache/" + self.__name + "_" + randomString])
-        tar.wait()
-        fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz")
-        dircount = 0
-        dirname = ""
-        for filename in os.listdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString):
-            if os.path.isdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + filename):
-                dirname = filename
-                dircount += 1
-        if dircount != 1:
-            return False
-        try:
-            self.__image = client.images.build(path=config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + dirname, rm=True, pull=True)[0]
+    def __build(self, url, download = True):
+        if download:
+            if not os.path.exists(config.servicepath + "buildcache"):
+                os.makedirs(config.servicepath + "buildcache")
+            randomString = ess.essentials.randomString(10)
             fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString)
+            fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz")
+            os.makedirs(config.servicepath + "buildcache/" + self.__name + "_" + randomString)
+            urllib.request.urlretrieve(url, config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz")
+            tar = Popen(["/bin/tar", "-zxf", config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz", "--directory", config.servicepath + "buildcache/" + self.__name + "_" + randomString])
+            tar.wait()
+            fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz")
+            dircount = 0
+            sourcePath = ""
+            for filename in os.listdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString):
+                if os.path.isdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + filename):
+                    sourcePath = config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + filename
+                    dircount += 1
+            if dircount != 1:
+                return False
+        else:
+            sourcePath = url
+        try:
+            self.__image = client.images.build(path=sourcePath, rm=True, pull=True)[0]
+            if download:
+                fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString)
             self.__status = 1
             return self.__image.id
         except docker.errors.BuildError:

modules/service.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # SchoolConnect Server-Manager - service class
-# © 2019 - 2020 Johannes Kreutz.
+# © 2019 - 2021 Johannes Kreutz.
 
 # Possible service wantings
 # running: Service is up and running
@@ -35,16 +35,15 @@
 import modules.container as container
 import modules.volume as volume
 import modules.network as network
-import modules.image as image
-import modules.envstore as envstore
 import modules.prune as prune
 import modules.servicedescription as description
 import modules.filesystem as fs
 import modules.builderThread as bt
 import modules.repository as repository
+from modules.envstore import envman
 
 # Manager objects
-env = envstore.envman()
+env = envman()
 
 # Class definition
 class service:
@@ -61,13 +60,15 @@ def __init__(self, name, firstinstall):
         self.__revertThread = None
         self.__deleteThread = None
         self.__rebuildThread = None
+        self.__localEnv = None
         if not firstinstall:
             # Read container configuration
             configFile = open(config.servicepath + name + "/config.json", "r")
             self.__config = json.loads(configFile.read())
             configFile.close()
             # Create service description object
             self.__desc = description.serviceDescription(False, self.__name, self.__config["actualVersion"])
+            self.__localEnv = envman(self.__name)
             # Create docker references for volumes
             for storedVolume in self.__config["volumes"]:
                 self.__volumes.append(volume.volume(True, storedVolume["id"], None))
@@ -76,9 +77,9 @@ def __init__(self, name, firstinstall):
                 self.networks.append(network.network(True, storedNetwork["id"], None, None))
             # Create docker references for containers
             for storedContainer in self.__config["containers"]["actual"]:
-                self.__containers["actual"].append(container.container(True, storedContainer["id"]))
+                self.__containers["actual"].append(container.container(True, storedContainer["id"], self.__name))
             for storedContainer in self.__config["containers"]["previous"]:
-                self.__containers["previous"].append(container.container(True, storedContainer["id"]))
+                self.__containers["previous"].append(container.container(True, storedContainer["id"], self.__name))
             # Check container status and turn containers to wanted status
             if self.__config["wanted"]:
                 for containerObject in self.__containers["actual"]:
@@ -116,6 +117,7 @@ def prepareBuild(self, name, url, version):
         self.__saveConfiguration()
         # Download service description
         self.__desc = description.serviceDescription(True, self.__name, self.__config["actualVersion"], url)
+        self.__localEnv = envman(self.__name)
         # Check if all required environment variables are set
         requiredVars = self.__requiredEnvironmentVariables()
         self.__config["status"] = "installPending"
@@ -129,11 +131,18 @@ def prepareBuild(self, name, url, version):
     def __requiredEnvironmentVariables(self):
         requiredVars = {}
         for var in self.__desc.getEnvironment():
-            if not env.doesKeyExist(var["name"]):
-                if "default" in var:
-                    env.storeValue(var["name"], var["default"], var["description"], var["mutable"])
-                else:
-                    requiredVars[var["name"]] = {"description":var["description"],"mutable":var["mutable"]}
+            if "private" in var and var["private"] == True:
+                if not self.__localEnv.doesKeyExist(var["name"]):
+                    if "default" in var:
+                        self.__localEnv.storeValue(var["name"], var["default"], var["description"], var["mutable"])
+                    else:
+                        requiredVars["[" + self.__name + "]" + var["name"]] = {"description":var["description"],"mutable":var["mutable"]}
+            else:
+                if not env.doesKeyExist(var["name"]):
+                    if "default" in var:
+                        env.storeValue(var["name"], var["default"], var["description"], var["mutable"])
+                    else:
+                        requiredVars[var["name"]] = {"description":var["description"],"mutable":var["mutable"]}
         if len(requiredVars) > 0:
             return requiredVars
         else:
@@ -161,7 +170,7 @@ def continueInstallation(self):
         # Add containers to queue
         self.__queueLock.acquire()
         for name in containerList:
-            containerObject = {"object":self.__desc.getContainerObject(name),"image":self.__desc.getImageSource(name),"volumeSource":None}
+            containerObject = {"object":self.__desc.getContainerObject(name),"image":self.__desc.getImageSource(name),"volumeSource":None,"name":self.__name}
             self.__buildQueue.put(containerObject)
         self.__queueLock.release()
         # Create and start threads
@@ -384,7 +393,7 @@ def revert(self):
         # Rename old containers back to original name
         for ct in self.__containers["actual"]:
             for name in names:
-                if name in ct.getName():
+                if name + "_" + self.__config["previousVersion"] == ct.getName():
                     ct.rename(name)
         # Read old service description
         self.__desc = description.serviceDescription(False, self.__name, self.__config["previousVersion"])