Merge pull request #42 from RainbowRevenge/master

ready for v2.0.1

Author: RainbowRevenge

CHANGELOG.md

@@ -1,9 +1,14 @@
 # Changelog
 
-## 2.0.1 (tbd)
+## 2.0.1 (25-02-2022)
 
 ### Added
 
+-   `flask create-upload-file` automatically create a upload file containing filepaths from a directory
+-   Polygon Task: Draw and modify polygons on an image.
+-   New examples for annotation protocols
+-   Freetext task allowing input of an arbitrary string
+
 ### Removed
 
 ### Changed
@@ -12,6 +17,11 @@
 -   node is only required for development from now on
     -   dist folder contains minified js and css
     -   when developing client side node is still needed for webpack
+-   transfer source code out of static folder
+    -   client source code is now in separate /client folder
+    -   moved package.json and webpack scripts to /client
+    -   npm scripts have to be started from /client
+    -   transpiled, minified js and css files and assets remain in /app/static/dist
 
 ## 2.0.0 (13-01-2022)
 

Dockerfile

@@ -9,20 +9,18 @@ HUMAN is an annotation server that stands for...
 
 # Try our [DEMO](http://human.lsv.uni-saarland.de)!
 
-
-
 ## Installation and Setup
 
 The only requirement is a working version of python 3.9.x. Using anaconda or [https://docs.conda.io/en/latest/miniconda.html](miniconda) for a python environment is highly recommended.
 
 The setup consists of 5 parts:
+
 1. [Python environment](#python-environment)
 2. [Database setup](#database-setup)
 3. [Annotation protocol](#annotation-protocol)
 4. [Deploy server](#deploy-server)
 5. [Add data](#add-data)
 
-
 ### Python environment
 
 First install the python environment.
@@ -71,20 +69,23 @@ flask run reset-annotations
 
 This will reset the annotations table in the database and is necessary to properly save annotations after a change in the protocol.
 
-
 ### Deploy server
 
 To run a server you have 2 possibilities:
 
 #### 1. Run locally on your machine
-Locally you can start a server  with
+
+Locally you can start a server with
+
 ```sh
-flask start
+flask run
 ```
+
 and visit http://127.0.0.1:5000 in your favorite browser and login with your admin account.
 However it is very ill advised to use this with an open port in a production environment.
 
 #### 2. Deploy in a production environment
+
 When running HUMAN in a production environment on a server we recommend using gunicorn (included in the environment). First, you should set a secure SECRET_KEY in config.py. The script `start_server.sh` should take care of starting the server.
 
 For docker refer to the [https://github.com/uds-lsv/human/wiki/Docker/](wiki).

README.md

@@ -60,11 +60,11 @@ def save(self, data=None, **kwargs):
 
 
     def write_to_db(self, **kwargs):
-        print('write to db', self.annotations)
-        self.to_start()
-        return
+        # print('write to db', self.annotations)
+        # self.to_start()
+        # return
         data = self.annotations
-        # copy of write to db
+        # copy of write to db in routes.py
         if not data:
                 return "No Annotations"
         else:
@@ -94,7 +94,7 @@ def write_to_db(self, **kwargs):
             # Unconditionally set. If the user completed an annotation, it was current_annotation.
             db.execute("UPDATE user SET current_annotation = 0 WHERE id = ?", (current_user.get_id(),))
 
-            # db.commit() # TODO comment in again
+            db.commit() # TODO comment in again
             # transition to start
             self.to_start()
 

app/automaton.py

@@ -22,6 +22,7 @@ def get_db():
             current_app.config['DATABASE'],
             detect_types=sqlite3.PARSE_DECLTYPES
         )
+        g.db.execute("PRAGMA foreign_keys=ON") # prevents removing users or data that has annotations
         g.db.row_factory = sqlite3.Row
 
     return g.db
@@ -41,6 +42,14 @@ def init_db():
     """Clear existing data and create new tables."""
     db = get_db()
 
+    # drop tables
+    # db.execute("PRAGMA foreign_keys=OFF") # allows removing users or data that has annotations
+    db.execute('DROP TABLE IF EXISTS annotations;')
+    db.execute('DROP TABLE IF EXISTS user;')
+    db.execute('DROP TABLE IF EXISTS data;')
+    db.execute('DROP TABLE IF EXISTS options;')
+
+    # load defaults for tables
     with current_app.open_resource('schema.sql') as f:
         db.executescript(f.read().decode('utf8'))
 
@@ -53,7 +62,12 @@ def columns_from_automaton():
         db.execute(f'ALTER TABLE annotations ADD COLUMN "{column}" text;')
     db.commit()
 
-def save_db(table):
+def save_db(table: str ):
+    """
+    Save the specified table as csv. 
+
+    If argument table="all" save all tables
+    """
     db = get_db()
     if table=="all":
         tables = db.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall()
@@ -92,6 +106,36 @@ def init_db_command():
 
     click.echo('Initialized the database.')
 
+
+@click.command('reset-annotations')
+@with_appcontext
+def reset_annotations_command():
+    """Update annotations table columns. Previous table has to be dropped for this change"""
+    if not os.path.isfile(current_app.config['DATABASE']):
+        init_db()
+    else:
+        while True:
+            create = input('This will drop the annotations table! It will be backed up in the ./output folder. y/n: ')
+            if create == 'no' or create == 'n':
+                return
+            elif create == 'yes' or create == 'y':
+                break
+        save_db('annotations')
+        reset_table('annotations')
+        columns_from_automaton()
+
+    click.echo('Initialized the database.')
+
+def reset_table(table):
+    """Drop specified table and load default table from schema.sql"""
+    db = get_db()
+    # drop table
+    db.execute(f'DROP TABLE IF EXISTS {table}')
+    # load defaults for dropped table(s)
+    with current_app.open_resource('schema.sql') as f:
+        db.executescript(f.read().decode('utf8'))
+    db.commit()
+
 @click.command('db-from-csv')
 @click.argument('csv')
 @click.argument('table')
@@ -107,7 +151,7 @@ def db_from_csv_command(csv, table):
 @with_appcontext
 def add_admin():
     """
-    Function that shows prompt for user to add an Admin user to the database.
+    Starts prompt for user to add an Admin user to the database.
     """
     db = get_db()
     db_cursor = db.cursor()
@@ -138,6 +182,7 @@ def add_admin():
 @click.option('-dataid', '--data_id')
 @with_appcontext
 def remove_all_annotation_for_data(data_id):
+    """Remove all annotations for a given data id"""
     db = get_db()
     db_cursor = db.cursor()
 
@@ -162,7 +207,6 @@ def remove_all_annotation_for_data(data_id):
     # db_cursor.execute("INSERT INTO user (username, email, given_name, surname, password, user_type, is_approved, annotated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
     # (username, email, given_name, surname, password_hash, user_type, is_approved, annotated))
 
-
     db.commit()
     db.close()
     click.echo("Successfully removed.")
@@ -171,6 +215,7 @@ def remove_all_annotation_for_data(data_id):
 @click.option('-id', '--annotation_id')
 @with_appcontext
 def remove_single_annotation(annotation_id):
+    """Remove a single annotation given the annotation id"""
     db = get_db()
     db_cursor = db.cursor()
     if annotation_id:
@@ -183,9 +228,11 @@ def remove_single_annotation(annotation_id):
     click.echo("Successfully removed.")
 
 def remove_annotation(db_cursor, annotation_id):
-
+    """Utility function to remove a annotation given a annotation id"""
+    # find annotation and data rows for annotation id
     annotation = db_cursor.execute("SELECT * from annotations WHERE id=?", (annotation_id,)).fetchone()
     data = db_cursor.execute("SELECT * from data WHERE id=?", (annotation["data_id"],)).fetchone()
+    # decrement annotation count in data table
     annotation_count = data['annotation_count']
     if annotation_count <= 0:
         annotation_count = 0
@@ -194,12 +241,57 @@ def remove_annotation(db_cursor, annotation_id):
     db_cursor.execute("UPDATE data SET annotation_count=? WHERE id=?",
         (annotation_count, annotation["data_id"]))
 
+    # remove annotation from user table
     user = db_cursor.execute("SELECT * from user WHERE id=?", (annotation["user_id"],)).fetchone()
     annotated = " ".join([anno for anno in user['annotated'].split() if int(anno) != annotation["data_id"]])
     db_cursor.execute("UPDATE user SET annotated=? WHERE id=?", (annotated, annotation["user_id"]))
     db_cursor.execute("DELETE FROM annotations WHERE id=?",
         (annotation_id,))
 
+@click.command('create-upload-file')
+@click.argument('path')
+@click.argument('output', default="upload.tsv",  )
+@click.option('-s', '--suffix', help="Include only files with suffix")
+@click.option('--fullpath', is_flag=True, help="Add full path from root to file: /root/to/file")
+@click.option('--includedir', is_flag=True, help="Add enclosing folder to file: parent/file")
+@with_appcontext
+def create_upload_file_command(path, output, suffix, fullpath, includedir):
+    """
+    Create an upload file containing all files in a path. Context and Meta fields are empty
+    
+    Arguments:
+    PATH: containing files to list in upload file;
+    OUTPUT: Path to output file.
+    """
+    if not os.path.isdir(path):
+        click.echo(f'{path} is not a valid path.')
+        return
+    if fullpath and includedir:
+        click.echo(f'only one of fullpath or includedir allowed')
+        return
+    # header
+    str_content = 'content\tcontext\tmeta\n'
+    str_join = '\t\t\n'
+    # default
+    output = (output if output else "upload.tsv")
+    suffix = (suffix if suffix else "")
+    parent_path = (os.path.basename(os.path.realpath(path)) if includedir else "")
+    parent_path = (os.path.realpath(path) if fullpath else parent_path)
+
+    # get filenames
+    filenames = [
+        os.path.join(parent_path, f) for f in os.listdir(path)
+        if os.path.isfile(os.path.join(path, f)) 
+        and (f.lower().endswith(suffix))
+        ]
+    str_content += str_join.join(filenames)
+
+    with open(output, 'w') as f:
+        f.write(str_content)
+    click.echo(f"Wrote into {output}")
+
+
+
 def init_app(app):
     """Register database functions with the Flask app. This is called by
     the application factory.
@@ -211,3 +303,5 @@ def init_app(app):
     app.cli.add_command(add_admin)  
     app.cli.add_command(remove_all_annotation_for_data)  
     app.cli.add_command(remove_single_annotation)  
+    app.cli.add_command(reset_annotations_command)  
+    app.cli.add_command(create_upload_file_command)  

app/db.py

@@ -766,7 +766,7 @@ def home():
     Route for home page
     """
     current_app.logger.debug("from / (home_page)")
-    return render_template('annotation_page.html', user=current_user.username, admin=current_user.admin)
+    return render_template('annotation_page.html', user=current_user.username, admin=current_user.admin, reverse=current_app.config['SIDEBAR'])
 
 
 ########## MISC Functions

app/routes.py

@@ -1,36 +1,31 @@
-DROP TABLE IF EXISTS user;
-DROP TABLE IF EXISTS data;
-DROP TABLE IF EXISTS annotations;
-DROP TABLE IF EXISTS options;
-
 CREATE TABLE IF NOT EXISTS user
 (
-id integer primary key autoincrement not null,
-username text not null,
-email text not null,
-given_name text not null,
-surname text not null,
-password text not null,
-user_type text not null,
-is_approved text not null,
-annotated text not null,
-current_annotation integer not null default 0,
+id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+username TEXT NOT NULL,
+email TEXT NOT NULL,
+given_name TEXT NOT NULL,
+surname TEXT NOT NULL,
+password TEXT NOT NULL,
+user_type TEXT NOT NULL,
+is_approved TEXT NOT NULL,
+annotated TEXT NOT NULL,
+current_annotation INTEGER NOT NULL DEFAULT 0,
 automaton blob);
 
 CREATE TABLE IF NOT EXISTS data
 (
-id integer primary key autoincrement not null,
-content text not null,
-context text,
-meta text,
-annotation_count integer not null default 0);
+id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+content TEXT NOT NULL,
+context TEXT,
+meta TEXT,
+annotation_count INTEGER NOT NULL DEFAULT 0);
 
 CREATE TABLE IF NOT EXISTS annotations
 (
-id integer primary key autoincrement not null,
-data_id integer not null,
-user_id integer not null,
-timestamp timestamp default (strftime('%Y-%m-%d %H:%M:%S', 'now', 'localtime')) not null,
+id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+data_id INTEGER NOT NULL REFERENCES data,
+user_id INTEGER NOT NULL REFERENCES user,
+timestamp timestamp DEFAULT (strftime('%Y-%m-%d %H:%M:%S', 'now', 'localtime')) NOT NULL,
 timings
 );