diff --git a/config.py b/config.py index ba065bd..be84d18 100755 --- a/config.py +++ b/config.py @@ -5,7 +5,7 @@ # Config values # Version number -servermanagerversion = "1.1.1" +servermanagerversion = "1.2.0" # file paths servicepath = "/var/lib/servermanager/services/" diff --git a/install.sh b/install.sh index 7bfcdb9..e00d723 100755 --- a/install.sh +++ b/install.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # SchoolConnect Server-Installer -# © 2019 Johannes Kreutz. +# © 2019 - 2021 Johannes Kreutz. -version='1.1.1' +version='1.2.0' # Check for root rights if [[ $EUID > 0 ]]; then diff --git a/modules/builderThread.py b/modules/builderThread.py index a70c175..92539d1 100644 --- a/modules/builderThread.py +++ b/modules/builderThread.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SchoolConnect Server-Manager - image builder thread class -# © 2019 Johannes Kreutz. +# © 2019 - 2021 Johannes Kreutz. # Include dependencies import threading @@ -36,6 +36,6 @@ def __buildImage(self, object): # Build a container with given image and description def __buildContainer(self, image, object): - containerObject = container.container(False, None) + containerObject = container.container(False, None, object["name"]) containerObject.create(image, object["object"], object["volumeSource"]) return containerObject diff --git a/modules/container.py b/modules/container.py index 6b66214..94e8a98 100755 --- a/modules/container.py +++ b/modules/container.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SchoolConnect Server-Manager - container class -# © 2019 Johannes Kreutz. +# © 2019 - 2021 Johannes Kreutz. # Container status codes # 0: Clean init @@ -17,18 +17,19 @@ # Include modules import config -import modules.envstore as envstore +from modules.envstore import envman # Manager objects -env = envstore.envman() +env = envman() # Create docker connection client = docker.from_env() # Class definition class container: - def __init__(self, exists, id): + def __init__(self, exists, id, name): self.__status = 0 + self.__localEnv = envman(name) if exists: try: self.__container = client.containers.get(container_id=id) @@ -145,7 +146,11 @@ def __createEnvironmentMap(self, object): environmentMap = {} if "environment" in object: for var in object["environment"]: - environmentMap[var] = env.getValue(var) + local = self.__localEnv.getValue(var) + if local is False: + environmentMap[var] = env.getValue(var) + else: + environmentMap[var] = local if object["name"] == "pc_admin": apitokenfile = open(config.configpath + config.apitokenfile, "r") environmentMap["APIKEY"] = apitokenfile.read() diff --git a/modules/envparser.py b/modules/envparser.py new file mode 100644 index 0000000..d7aaacf --- /dev/null +++ b/modules/envparser.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +# SchoolConnect Server-Manager - env file parser (rewritten to use the comments above a key-value pair) +# © 2021 Johannes Kreutz. + +# Class definition +class envParser: + def __init__(self, path): + self.__path = path + + # Return the environment object for the given file + def getEnvironment(self): + envList = [] + with open(self.__path, "r") as f: + lastComment = "" + for line in f.readlines(): + if line.startswith("#"): + lastComment = line.strip() + else: + if "=" in line: + l = line.split("=") + mutable = True if lastComment.endswith(" MU") or lastComment.endswith(" UM") or lastComment.endswith(" M") else False + useValue = True if lastComment.endswith(" MU") or lastComment.endswith(" UM") or lastComment.endswith(" U") else False + private = False if lastComment.endswith(" S") else True + if lastComment.endswith(" MU") or lastComment.endswith(" UM"): + lastComment = lastComment[:-2] + if lastComment.endswith(" U") or lastComment.endswith(" M") or lastComment.endswith(" S"): + lastComment = lastComment[:-1] + if lastComment.startswith("#"): + lastComment = lastComment[1:] + lastComment = lastComment.strip() + env = { + "name": l[0].strip(), + "description": lastComment, + "mutable": mutable, + "private": private, + } + if useValue: + env["default"] = l[1].strip() + envList.append(env) + lastComment = "" + elif len(line) <= 1: + continue + else: + return None + return envList diff --git a/modules/envstore.py b/modules/envstore.py index c120b41..d74f9ed 100755 --- a/modules/envstore.py +++ b/modules/envstore.py @@ -13,21 +13,28 @@ # Class definition class envman: - def __init__(self): - if not os.path.exists(config.configpath + "env.json"): - self.__storage = {} + def __init__(self, service = None): + if service is None: + self.__path = config.configpath + "env.json" + else: + self.__path = config.servicepath + service + "/env.json" + self.__storage = {} + if not os.path.exists(self.__path): self.write() - self.updateLocalIp() + else: + self.load() + if service is None: + self.updateLocalIp() # Read stored environment variables and their descriptions from configuration files def load(self): - envfile = open(config.configpath + "env.json", "r") + envfile = open(self.__path, "r") self.__storage = json.loads(envfile.read()) envfile.close() # Writes actual environment variables and their descriptions to the storage files def write(self): - envfile = open(config.configpath + "env.json", "w") + envfile = open(self.__path, "w") envfile.write(json.dumps(self.__storage, sort_keys=True, indent=4)) envfile.close() diff --git a/modules/image.py b/modules/image.py index 75f202d..50397ad 100755 --- a/modules/image.py +++ b/modules/image.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SchoolConnect Server-Manager - image class -# © 2019 Johannes Kreutz. +# © 2019 - 2021 Johannes Kreutz. # Image status definitions # 0: Clean init @@ -50,8 +50,10 @@ def create(self, object, wantedVersion): self.__wantedVersion = wantedVersion if "prebuilt" in self.__wantedVersion: return self.__pull(self.__wantedVersion["prebuilt"]["name"] + ":" + self.__wantedVersion["prebuilt"]["version"]) - else: + elif "url" in self.__wantedVersion: return self.__build(self.__wantedVersion["url"]) + else: + return self.__build(self.__wantedVersion["path"], False) # Pulls an image from the docker hub def __pull(self, name): @@ -64,28 +66,32 @@ def __pull(self, name): return False # Builds an image from a given url - def __build(self, url): - if not os.path.exists(config.servicepath + "buildcache"): - os.makedirs(config.servicepath + "buildcache") - randomString = ess.essentials.randomString(10) - fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString) - fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz") - os.makedirs(config.servicepath + "buildcache/" + self.__name + "_" + randomString) - urllib.request.urlretrieve(url, config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz") - tar = Popen(["/bin/tar", "-zxf", config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz", "--directory", config.servicepath + "buildcache/" + self.__name + "_" + randomString]) - tar.wait() - fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz") - dircount = 0 - dirname = "" - for filename in os.listdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString): - if os.path.isdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + filename): - dirname = filename - dircount += 1 - if dircount != 1: - return False - try: - self.__image = client.images.build(path=config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + dirname, rm=True, pull=True)[0] + def __build(self, url, download = True): + if download: + if not os.path.exists(config.servicepath + "buildcache"): + os.makedirs(config.servicepath + "buildcache") + randomString = ess.essentials.randomString(10) fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString) + fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz") + os.makedirs(config.servicepath + "buildcache/" + self.__name + "_" + randomString) + urllib.request.urlretrieve(url, config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz") + tar = Popen(["/bin/tar", "-zxf", config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz", "--directory", config.servicepath + "buildcache/" + self.__name + "_" + randomString]) + tar.wait() + fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString + ".tar.gz") + dircount = 0 + sourcePath = "" + for filename in os.listdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString): + if os.path.isdir(config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + filename): + sourcePath = config.servicepath + "buildcache/" + self.__name + "_" + randomString + "/" + filename + dircount += 1 + if dircount != 1: + return False + else: + sourcePath = url + try: + self.__image = client.images.build(path=sourcePath, rm=True, pull=True)[0] + if download: + fs.filesystem.removeElement(config.servicepath + "buildcache/" + self.__name + "_" + randomString) self.__status = 1 return self.__image.id except docker.errors.BuildError: diff --git a/modules/service.py b/modules/service.py index 947163a..31ba415 100755 --- a/modules/service.py +++ b/modules/service.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SchoolConnect Server-Manager - service class -# © 2019 - 2020 Johannes Kreutz. +# © 2019 - 2021 Johannes Kreutz. # Possible service wantings # running: Service is up and running @@ -35,16 +35,15 @@ import modules.container as container import modules.volume as volume import modules.network as network -import modules.image as image -import modules.envstore as envstore import modules.prune as prune import modules.servicedescription as description import modules.filesystem as fs import modules.builderThread as bt import modules.repository as repository +from modules.envstore import envman # Manager objects -env = envstore.envman() +env = envman() # Class definition class service: @@ -61,6 +60,7 @@ def __init__(self, name, firstinstall): self.__revertThread = None self.__deleteThread = None self.__rebuildThread = None + self.__localEnv = None if not firstinstall: # Read container configuration configFile = open(config.servicepath + name + "/config.json", "r") @@ -68,6 +68,7 @@ def __init__(self, name, firstinstall): configFile.close() # Create service description object self.__desc = description.serviceDescription(False, self.__name, self.__config["actualVersion"]) + self.__localEnv = envman(self.__name) # Create docker references for volumes for storedVolume in self.__config["volumes"]: self.__volumes.append(volume.volume(True, storedVolume["id"], None)) @@ -76,9 +77,9 @@ def __init__(self, name, firstinstall): self.networks.append(network.network(True, storedNetwork["id"], None, None)) # Create docker references for containers for storedContainer in self.__config["containers"]["actual"]: - self.__containers["actual"].append(container.container(True, storedContainer["id"])) + self.__containers["actual"].append(container.container(True, storedContainer["id"], self.__name)) for storedContainer in self.__config["containers"]["previous"]: - self.__containers["previous"].append(container.container(True, storedContainer["id"])) + self.__containers["previous"].append(container.container(True, storedContainer["id"], self.__name)) # Check container status and turn containers to wanted status if self.__config["wanted"]: for containerObject in self.__containers["actual"]: @@ -116,6 +117,7 @@ def prepareBuild(self, name, url, version): self.__saveConfiguration() # Download service description self.__desc = description.serviceDescription(True, self.__name, self.__config["actualVersion"], url) + self.__localEnv = envman(self.__name) # Check if all required environment variables are set requiredVars = self.__requiredEnvironmentVariables() self.__config["status"] = "installPending" @@ -129,11 +131,18 @@ def prepareBuild(self, name, url, version): def __requiredEnvironmentVariables(self): requiredVars = {} for var in self.__desc.getEnvironment(): - if not env.doesKeyExist(var["name"]): - if "default" in var: - env.storeValue(var["name"], var["default"], var["description"], var["mutable"]) - else: - requiredVars[var["name"]] = {"description":var["description"],"mutable":var["mutable"]} + if "private" in var and var["private"] == True: + if not self.__localEnv.doesKeyExist(var["name"]): + if "default" in var: + self.__localEnv.storeValue(var["name"], var["default"], var["description"], var["mutable"]) + else: + requiredVars["[" + self.__name + "]" + var["name"]] = {"description":var["description"],"mutable":var["mutable"]} + else: + if not env.doesKeyExist(var["name"]): + if "default" in var: + env.storeValue(var["name"], var["default"], var["description"], var["mutable"]) + else: + requiredVars[var["name"]] = {"description":var["description"],"mutable":var["mutable"]} if len(requiredVars) > 0: return requiredVars else: @@ -161,7 +170,7 @@ def continueInstallation(self): # Add containers to queue self.__queueLock.acquire() for name in containerList: - containerObject = {"object":self.__desc.getContainerObject(name),"image":self.__desc.getImageSource(name),"volumeSource":None} + containerObject = {"object":self.__desc.getContainerObject(name),"image":self.__desc.getImageSource(name),"volumeSource":None,"name":self.__name} self.__buildQueue.put(containerObject) self.__queueLock.release() # Create and start threads @@ -384,7 +393,7 @@ def revert(self): # Rename old containers back to original name for ct in self.__containers["actual"]: for name in names: - if name in ct.getName(): + if name + "_" + self.__config["previousVersion"] == ct.getName(): ct.rename(name) # Read old service description self.__desc = description.serviceDescription(False, self.__name, self.__config["previousVersion"]) diff --git a/modules/servicedescription.py b/modules/servicedescription.py index 9bb1fc9..d50bbf5 100644 --- a/modules/servicedescription.py +++ b/modules/servicedescription.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 # SchoolConnect Server-Manager - service description class -# © 2019 Johannes Kreutz. +# © 2019 - 2021 Johannes Kreutz. # Include dependencies import json import urllib.request import os +import yaml +from subprocess import Popen # Include modules import config import modules.repository as repo import modules.filesystem as fs +from modules.envparser import envParser # Manager objects repo = repo.repository() @@ -24,22 +27,41 @@ def __init__(self, firstsetup, name, version, url = ""): if firstsetup: self.loadDescriptionFile(self.__version) else: + if os.path.exists(config.servicepath + self.__name + "/" + version + "/docker-compose.yml"): + self.dockerComposeFile = True + else: + self.dockerComposeFile = False self.readDescriptionFile() # FILE MANAGEMENT # Loads a description file def loadDescriptionFile(self, version): url = repo.getUrl(self.__name, version) - fs.filesystem.removeElement(config.servicepath + self.__name + "/service_" + version + ".json") - urllib.request.urlretrieve(url, config.servicepath + self.__name + "/service_" + version + ".json") + if url.endswith(".json"): + self.dockerComposeFile = False + localPath = config.servicepath + self.__name + "/service_" + version + ".json" + else: + self.dockerComposeFile = True + localPath = config.servicepath + self.__name + "/" + version + ".tar.gz" + fs.filesystem.removeElement(localPath) + urllib.request.urlretrieve(url, localPath) self.__version = version + if self.dockerComposeFile: + fs.filesystem.removeElement(config.servicepath + self.__name + "/" + version) + os.makedirs(config.servicepath + self.__name + "/" + version) + tar = Popen(["/bin/tar", "-zxf", config.servicepath + self.__name + "/" + version + ".tar.gz", "--directory", config.servicepath + self.__name + "/" + version, "--strip-components", "1"]) + tar.wait() + fs.filesystem.removeElement(localPath) self.readDescriptionFile() # Reads a locally stored service file def readDescriptionFile(self): - serviceFile = open(config.servicepath + self.__name + "/service_" + self.__version + ".json", "r") - self.__desc = json.loads(serviceFile.read()) - serviceFile.close() + if self.dockerComposeFile: + with open(config.servicepath + self.__name + "/" + self.__version + "/docker-compose.yml", "r") as serviceFile: + self.__desc = yaml.safe_load(serviceFile.read()) + else: + with open(config.servicepath + self.__name + "/service_" + self.__version + ".json", "r") as serviceFile: + self.__desc = json.loads(serviceFile.read()) # Reload service description def checkForUpdate(self): @@ -56,14 +78,21 @@ def executeUpdate(self, version): # Removes an old locally stored file version def removeOldFile(self, version): - os.remove(config.servicepath + self.__name + "/service_" + version + ".json") + if self.dockerComposeFile: + os.remove(config.servicepath + self.__name + "/" + self.__version) + else: + os.remove(config.servicepath + self.__name + "/service_" + version + ".json") # VALUE GETTERS # Returns a list of all containers def getContainers(self): list = [] - for container in self.__desc["containers"]: - list.append(container["name"]) + if self.dockerComposeFile: + for name, container in self.__desc["services"].items(): + list.append(name) + else: + for container in self.__desc["containers"]: + list.append(container["name"]) return list # Returns the number of containers @@ -72,22 +101,54 @@ def getContainerCount(self): # Returns a container description for the given name def getContainerObject(self, name): - for container in self.__desc["containers"]: + for container in self.__getContainerObjectList(): if container["name"] == name: return container return False # Returns an array of all defined networks def getNetworks(self): - return self.__desc["networks"] + if self.dockerComposeFile: + networks = [] + for name, network in self.__desc["networks"].items(): + internal = True if network is not None and "internal" in network else False + networks.append({ + "name": name, + "internal": internal, + }) + return networks + else: + return self.__desc["networks"] # Returns an array of all defined volumes def getVolumes(self): - return self.__desc["volumes"] + if self.dockerComposeFile: + volumes = [] + for name, volume in self.__desc["volumes"].items(): + volumes.append({ + "name": name, + }) + return volumes + else: + return self.__desc["volumes"] # Returns an array of all defined environment variables def getEnvironment(self): - return self.__desc["environment"] + if self.dockerComposeFile: + environment = [] + names = [] # Cache names we already have + for name, container in self.__desc["services"].items(): + if "env_file" in container: + for file in container["env_file"]: + filename = file[2:] if file.startswith("./") else file + p = envParser(config.servicepath + self.__name + "/" + self.__version + "/" + filename) + for var in p.getEnvironment(): + if not var["name"] in names: + names.append(var["name"]) + environment.append(var) + return environment + else: + return self.__desc["environment"] # Get image source def getImageSource(self, name): @@ -95,6 +156,72 @@ def getImageSource(self, name): container = self.getContainerObject(name) if "prebuilt" in container: response["prebuilt"] = container["prebuilt"] - else: + elif "url" in container: response["url"] = container["url"] + else: + response["path"] = container["path"] return response + + # PRIVATE HELPERS + # Get a list with all containet objects + def __getContainerObjectList(self): + if self.dockerComposeFile: + list = [] + for name, container in self.__desc["services"].items(): + c = { + "name": name, + "hostname": name, + "ports": [], + "networks": [], + "volumes": [], + "environment": [] + } + # Translate image source + if "image" in container: + source = container["image"] + if ":" in source: + parts = source.split(":") + c["prebuilt"] = { + "name": parts[0], + "version": parts[1], + } + else: + c["prebuilt"] = { + "name": container["image"], + "version": "latest", + } + else: + c["path"] = config.servicepath + self.__name + "/" + self.__version + "/" + container["build"] + # Translate ports + if "ports" in container: + for port in container["ports"]: + parts = port.split(":") + c["ports"].append({ + "external": parts[0], + "internal": parts[1], + }) + # Translate networks + if "networks" in container: + for network in container["networks"]: + c["networks"].append({ + "name": network, + }) + # Translate volumes + if "volumes" in container: + for volume in container["volumes"]: + parts = volume.split(":") + c["volumes"].append({ + "name": parts[0], + "mountpoint": parts[1], + }) + # Translate environment parameters + if "env_file" in container: + for envFile in container["env_file"]: + filename = envFile[2:] if envFile.startswith("./") else envFile + p = envParser(config.servicepath + self.__name + "/" + self.__version + "/" + filename) + for var in p.getEnvironment(): + c["environment"].append(var["name"]) + list.append(c) + return list + else: + return self.__desc["containers"] diff --git a/servermanager.py b/servermanager.py index 25c866b..1482533 100755 --- a/servermanager.py +++ b/servermanager.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SchoolConnect Server-Manager -# © 2019 - 2020 Johannes Kreutz. +# © 2019 - 2021 Johannes Kreutz. # Include dependencies import sys @@ -21,13 +21,13 @@ import modules.repository as repository import modules.service as service import modules.network as network -import modules.envstore as envstore import modules.managerupdate as update import modules.essentials as ess +from modules.envstore import envman # Manager objects repo = repository.repository() -env = envstore.envman() +env = envman() api = Flask(__name__) # Variables @@ -353,7 +353,13 @@ def setBranch(): def listEnv(): data = request.form if data.get("apikey") == getApiKey(): - return env.getJson() + main = json.loads(env.getJson()) + for service in services: + localEnv = envman(service.getName()) + data = json.loads(localEnv.getJson()) + for id, content in data.items(): + main["[" + service.getName() + "]" + id] = content + return json.dumps(main) else: return json.dumps({"error":"ERR_AUTH"}) # Store a new environment variable @@ -362,7 +368,13 @@ def storeEnv(): data = request.form if data.get("apikey") == getApiKey(): for key, entry in json.loads(data.get("data")).items(): - env.storeValue(key, entry["value"], entry["description"], entry["mutable"]) + if key.startswith("["): + parts = key.split("]") + serviceName = parts[0][1:] + localEnv = envman(serviceName) + localEnv.storeValue(parts[1], entry["value"], entry["description"], entry["mutable"]) + else: + env.storeValue(key, entry["value"], entry["description"], entry["mutable"]) return json.dumps({"result":"SUCCESS"}) else: return json.dumps({"error":"ERR_AUTH"})