Merge pull request #1322 from betagouv/1270-ajout-champs-open-data-de…

…clarations

1270 Ajout champs supplémentaires pour le jeu de données en open data "declarations"

api/serializers/declaration.py

@@ -313,7 +313,6 @@ class OpenDataDeclarationSerializer(serializers.ModelSerializer):
 
     declared_plants = serializers.SerializerMethodField()
     declared_microorganisms = serializers.SerializerMethodField()
-    declared_ingredients = serializers.SerializerMethodField()
     declared_substances = serializers.SerializerMethodField()
 
     modification_date = serializers.DateTimeField(format="%Y-%m-%d")
@@ -329,10 +328,12 @@ class Meta:
             "gamme",
             "article",
             "galenic_formulation",
+            "daily_recommended_dose",
+            "instructions",
+            "warning",
             "declared_plants",
             "declared_microorganisms",
             "declared_substances",
-            "declared_ingredients",
             "modification_date",
         )
         read_only_fields = fields
@@ -363,19 +364,38 @@ def get_declared_plants(self, obj):
                 "partie": declared_plant.used_part.name,
                 "preparation": declared_plant.preparation.name,
                 "quantité_par_djr": declared_plant.quantity,
-                "unit": declared_plant.unit,
+                "unite": declared_plant.unit.name,
             }
-            for declared_plant in obj.declared_plants.all()
+            if declared_plant.unit
+            else {}
+            for declared_plant in obj.declared_plants.filter(active=True)
         ]
 
     def get_declared_microorganisms(self, obj):
-        return {"nom": obj.declared_plants.name}
-
-    def get_declared_ingredients(self, obj):
-        return {"nom": obj.declared_plants.name}
+        return [
+            {
+                "genre": declared_microorganism.microorganism.genus,
+                "espece": declared_microorganism.microorganism.species,
+                "souche": declared_microorganism.strain
+                if declared_microorganism.strain
+                else None,  # elle est normalement obligatoire mais quelques entrées ont pu être rentrées avant le required
+                "quantité_par_djr": declared_microorganism.quantity if declared_microorganism.activated else None,
+                "inactive": not declared_microorganism.activated,
+            }
+            for declared_microorganism in obj.declared_microorganisms.all()
+        ]
 
     def get_declared_substances(self, obj):
-        return {"nom": obj.declared_plants.name}
+        return [
+            {
+                "nom": declared_substance.substance.name,
+                "quantité_par_djr": declared_substance.quantity,
+                "unite": declared_substance.unit.name,
+            }
+            if declared_substance.substance.name and declared_substance.quantity and declared_substance.unit
+            else {}
+            for declared_substance in obj.declared_substances.all()
+        ]
 
 
 class DeclarationSerializer(serializers.ModelSerializer):

data/etl/extractor.py

@@ -30,6 +30,7 @@ def __init__(self):
         self.schema = None
         self.schema_url = ""
         self.dataset_name = ""
+        self.columns = []
 
     def get_schema(self):
         return self.schema
@@ -51,6 +52,13 @@ def filter_dataframe_with_schema_cols(self):
 
     def clean_dataset(self):
         self.df = self.df.loc[:, ~self.df.columns.duplicated()]
+        ## Code temporaire en attendant d'avoir tous les champs du schéma
+        columns_to_keep = []
+        for col in self.columns:
+            if col in self.df.columns:
+                columns_to_keep.append(col)
+        # ---------------------------------------------------
+        self.df = self.df[columns_to_keep]
         self.filter_dataframe_with_schema_cols()
 
     def is_valid(self) -> bool:
@@ -97,22 +105,6 @@ def __init__(self):
         self.df = None
         self.columns = [i["name"] for i in self.schema["fields"]]
 
-        self.columns_mapper = {
-            "id": "id",
-            "status": "decision",
-            "name": "nom_commercial",
-            "brand": "marque",
-            "gamme": "gamme",
-            "company": "responsable_mise_sur_marche",
-            "article": "article",
-            "galenic_formulation": "forme_galenique",
-            "declared_plants": "plantes",
-            "declared_microorganisms": "micro_organismes",
-            "declared_ingredients": "ingredients_inactifs",
-            "declared_substances": "substances",
-            "modification_date": "date_decision",  #  Warning : Se baser sur la du snapshot d'autorisation si la plateforme Compl'Alim permet d'editer la déclaration (ex: abandon)
-        }
-
     def extract_dataset(self):
         open_data_view = OpenDataDeclarationsListView()
         queryset = open_data_view.get_queryset()

data/etl/transformer_loader.py

@@ -43,8 +43,27 @@ def load_dataset(self):
 
 
 class ETL_OPEN_DATA_DECLARATIONS(DECLARATIONS, OPEN_DATA):
+    def __init__(self):
+        super().__init__()
+        self.columns_mapper = {
+            "id": "id",
+            "status": "decision",
+            "name": "nom_commercial",
+            "brand": "marque",
+            "gamme": "gamme",
+            "article": "article",
+            "galenic_formulation": "forme_galenique",
+            "daily_recommended_dose": "dose_journaliere",
+            "instructions": "mode_emploi",
+            "warning": "mises_en_garde",
+            "declared_plants": "plantes",
+            "declared_microorganisms": "micro_organismes",
+            "declared_substances": "substances",
+            "modification_date": "date_decision",  #  Warning : Se baser sur la du snapshot d'autorisation si la plateforme Compl'Alim permet d'editer la déclaration (ex: abandon)
+        }
+
     def transform_dataset(self):
         self.df = self.df.rename(columns=self.columns_mapper)
-        self.df["responsable_mise_sur_marche"] = self.df["responsable_mise_sur_marche"].apply(lambda x: x[0])
-        self.df["siret_responsable_mise_sur_marche"] = self.df["responsable_mise_sur_marche"].apply(lambda x: x[1])
+        self.df["responsable_mise_sur_marche"] = self.df["company"].apply(lambda x: x[0])
+        self.df["siret_responsable_mise_sur_marche"] = self.df["company"].apply(lambda x: x[1])
         self.clean_dataset()

data/schemas/schema_declarations.json

@@ -68,20 +68,21 @@
         "constraints": {
           "required": "True"
         },
-        "description": "Nom de l'entreprise ayant fabriqué le complément alimentaire",
-        "example": "",
-        "name": "nom_fabricant",
-        "title": "Nom fabricant",
+        "description": "Nom de l'entreprise responsable de la mise sur le marché du complément alimentaire",
+        "example": "Compl Corp",
+        "name": "responsable_mise_sur_marche",
+        "title": "Responsable de la mise sur le marché",
         "type": "string"
       },
       {
         "constraints": {
           "pattern": "^[0-9]{14}$",
           "required": true
         },
+        "description": "Siret de l'entreprise responsable de la mise sur le marché du complément alimentaire",
         "example": "11007001800012",
-        "name": "siret_fabricant",
-        "title": "Siret fabricant",
+        "name": "siret_responsable_mise_sur_marche",
+        "title": "Siret responsable de la mise sur le marché",
         "type": "integer"
       },
       {