Catch up dev.

.gitignore

@@ -4,3 +4,4 @@
 .idea
 __pycache__/
 docs/.DS_Store
+env

README.md

@@ -24,21 +24,26 @@ a language created at Google earlier.
 ## Why?
 
 Logica is for engineers, data scientists and other specialists who want to use
-logic programming syntax when writing queries and pipelines to run on
-[BigQuery](https://cloud.google.com/bigquery).
+logic programming syntax when writing queries and pipelines for databases and datawarehouses.
+Logica programs run on
+[BigQuery](https://cloud.google.com/bigquery), [Postgres](https://postgresql.org) and [SQLite](https://www.sqlite.org/).
 
-Logica compiles to StandardSQL and gives you access to the power of BigQuery
-engine with the convenience of logic programming syntax. This is useful because
-BigQuery is magnitudes more powerful than state of the art native
-logic programming engines.
+Logica compiles to SQL and gives you access to the power of SQL ecosystem
+with the convenience of logic programming syntax. 
+
+This is useful because
+SQL enginers are magnitudes more powerful than state of the art native
+logic programming engines. For example, BigQuery is a distributed datawarehouse and thus logic programs written
+in Logica can be easily parallelized onto thousands of servers. Postgres and SQLite are among most popular databases, they are
+capable of processing substantial volumes of data right on your machine.
 
 We encourage you to try Logica, especially if
 
 *   you already use logic programming and need more computational power, **or**
-*   you use SQL, but feel unsatisfied about its readability, **or**
+*   you already have data in BigQuery, PostgreSQL or SQLite, **or**
 *   you want to learn logic programming and apply it to processing of Big Data.
 
-In the future we plan to support more SQL dialects and engines.
+Support for more SQL dialects and engines is coming in the future.
 
 ## I have not heard of logic programming. What is it?
 
@@ -140,18 +145,14 @@ Find prime numbers less than 30.
 
 Program `primes.l`:
 ```
-# Define natural numbers from 1 to 29.
-N(x) :- x in Range(30);
-# Define primes.
-Prime(prime: x) :-
-  N(x),
-  x > 1,
-  ~(
-    N(y),
-    y > 1,
-    y != x,
-    x % y == 0
-  );
+# Define numbers 1 to 30.
+Number(x + 1) :- x in Range(30);
+
+# Defining composite numbers.
+Composite(a * b) distinct :- Number(a), Number(b), a > 1, b > 1;
+
+# Defining primes as "not composite".
+Prime(n) distinct :- Number(n), n > 1, ~Composite(n);
 ```
 
 Running `primes.l`
@@ -173,6 +174,49 @@ $ logica primes.l run Prime
 +-------+
 ```
 
+### Cities with largest beer variety
+
+Let's use beer variety dataset from [plotly](https://github.com/plotly/datasets/blob/master/beers.csv).
+
+Let us find top 5 states with largest variety of beers. In each state we will pick city with the largest
+variety in the state.
+
+Program `beer.l`:
+
+```
+@Engine("duckdb");
+
+@Ground(Beer);
+Beer(..r) :- 
+    `('https://github.com/plotly/datasets/blob/master/beers.csv?raw=true')`(..r);
+
+BeersInState(state) += 1 :- Beer(state:);
+BeersInCity(state, city) += 1 :- Beer(state:, city:);
+
+ArgMax5(x) = ArgMaxK(x, 5);
+BestCityForBeer(state:, city:,
+                city_beers: BeersInCity(state, city),
+                state_beers: BeersInState(state)) :-
+    state in ArgMax5{s -> BeersInState(s)},
+    city = ArgMax{c -> BeersInCity(state, c)};
+```
+
+Running `beer.l`:
+
+```
+# logica beer.l run BestCityForBeer
++-------+--------------+------------+-------------+
+| state | city         | city_beers | state_beers |
++-------+--------------+------------+-------------+
+| IN    | Indianapolis | 43         | 139         |
+| CO    | Boulder      | 41         | 265         |
+| CA    | San Diego    | 42         | 183         |
+| TX    | Austin       | 25         | 130         |
+| MI    | Grand Rapids | 66         | 162         |
++-------+--------------+------------+-------------+
+```
+
+<!--
 ### News mentions
 
 Who was mentioned in the news in 2020 the most?
@@ -210,7 +254,7 @@ $ logica mentions.l run Mentions
 
 Note that cities of Los Angeles and Las Vegas are mentioned in this table due to known
 missclasification issue in the GDELT data analysis.
-
+--> 
 ## Feedback
 
 Feel free to create [github issues](https://github.com/EvgSkv/logica/issues)

colab_logica.py

@@ -39,9 +39,11 @@
 import os
 
 import pandas
+import duckdb
 
 from .parser_py import parse
 from .common import sqlite3_logica
+from .common import duckdb_logica
 
 BQ_READY = True  # By default.
 
@@ -98,7 +100,14 @@ def SetDbConnection(connection):
   global DB_CONNECTION
   DB_CONNECTION = connection
 
+REMEMBERED_PREVIOUS_MODE = None
 def ConnectToPostgres(mode='interactive'):
+  global REMEMBERED_PREVIOUS_MODE
+  if mode == 'reconnect':
+    mode = REMEMBERED_PREVIOUS_MODE
+    print('Reconnecting to Postgres.')
+  else:
+    REMEMBERED_PREVIOUS_MODE = mode
   connection = psql_logica.ConnectToPostgres(mode)
   SetDbConnection(connection)
   global DEFAULT_ENGINE
@@ -157,6 +166,12 @@ def ParseList(line):
     predicates = [p.strip() for p in line.split(',')]
   return predicates
 
+def ParseListAndMaybeFile(line):
+  if '>' in line:
+    predicate_list_str, storage_file = line.split('>')
+    return ParseList(predicate_list_str.strip()), storage_file.strip()
+  else:
+    return ParseList(line), None
 
 def RunSQL(sql, engine, connection=None, is_final=False):
   if engine == 'bigquery':
@@ -172,6 +187,18 @@ def RunSQL(sql, engine, connection=None, is_final=False):
       return df
     else:
       psql_logica.PostgresExecute(sql, connection)
+  elif engine == 'duckdb':
+    if is_final:
+      df = connection.sql(sql).df()
+      for c in df.columns:
+        if df.dtypes[c] == 'float64':
+          if df[c].isna().values.any():
+            return df
+          if set(df[c] - df[c].astype(int)) == {0.0}:
+            df[c] = df[c].astype(int)
+      return df
+    else:
+      connection.sql(sql)
   elif engine == 'sqlite':
     try:
       if is_final:
@@ -207,11 +234,27 @@ def __init__(self):
   def __call__(self, sql, engine, is_final):
     return RunSQL(sql, engine, self.connection, is_final)
 
+class DuckdbRunner(object):
+  def __init__(self):
+    global DB_CONNECTION
+    if not DB_CONNECTION:
+      DB_CONNECTION = duckdb.connect()
+    self.connection = DB_CONNECTION
+
+  def  __call__(self, sql, engine, is_final):
+    return RunSQL(sql, engine, self.connection, is_final)
+
 
 class PostgresRunner(object):
   def __init__(self):
     global DB_CONNECTION
     global DB_ENGINE
+    if not DB_CONNECTION:
+      try:
+        ConnectToLocalPostgres()  
+      except:
+        pass
+
     if not DB_CONNECTION:
       print("Assuming this is running on Google CoLab in a temporary")
       print("environment.")
@@ -225,6 +268,10 @@ def __init__(self):
     self.connection = DB_CONNECTION
 
   def  __call__(self, sql, engine, is_final):
+    global DB_CONNECTION
+    if self.connection and self.connection.closed and REMEMBERED_PREVIOUS_MODE:
+      ConnectToPostgres('reconnect')
+      self.connection = DB_CONNECTION
     return RunSQL(sql, engine, self.connection, is_final)
 
 
@@ -234,7 +281,7 @@ def ShowError(error_text):
 
 def Logica(line, cell, run_query):
   """Running Logica predicates and storing results."""
-  predicates = ParseList(line)
+  predicates, maybe_storage_file = ParseListAndMaybeFile(line)
   if not predicates:
     ShowError('No predicates to run.')
     return
@@ -257,7 +304,6 @@ def Logica(line, cell, run_query):
   except infer.TypeErrorCaughtException as e:
     e.ShowMessage()
     return
-
   engine = program.annotations.Engine()
 
   if engine == 'bigquery' and not BQ_READY:
@@ -272,13 +318,16 @@ def Logica(line, cell, run_query):
 
   bar = TabBar(predicates + ['(Log)'])
   logs_idx = len(predicates)
-
   executions = []
   sub_bars = []
   ip = IPython.get_ipython()
   for idx, predicate in enumerate(predicates):
     with bar.output_to(logs_idx):
       try:
+        if storage_file_name := maybe_storage_file:
+          with open(storage_file_name, 'w') as storage_file:
+            storage_file.write(cell)
+            print('\x1B[3mProgram saved to %s.\x1B[0m' % storage_file_name)
         sql = program.FormattedPredicateSql(predicate)
         executions.append(program.execution)
         ip.push({predicate + '_sql': sql})
@@ -307,13 +356,15 @@ def Logica(line, cell, run_query):
       sql_runner = SqliteRunner()
     elif engine == 'psql':
       sql_runner = PostgresRunner()
+    elif engine == 'duckdb':
+      sql_runner = DuckdbRunner() 
     elif engine == 'bigquery':
       EnsureAuthenticatedUser()
       sql_runner = RunSQL
     else:
       raise Exception('Logica only supports BigQuery, PostgreSQL and SQLite '
                       'for now.')   
-    try:                  
+    try:
       result_map = concertina_lib.ExecuteLogicaProgram(
         executions, sql_runner=sql_runner, sql_engine=engine,
         display_mode=DISPLAY_MODE)
@@ -337,6 +388,19 @@ def Logica(line, cell, run_query):
           print('The query was not run.')
       print(' ') # To activate the tabbar.
 
+
+def ConnectToLocalPostgres():
+  import psycopg2
+  connection = psycopg2.connect(host='localhost', database='logica', user='logica', password='logica')
+  connection.autocommit = True
+
+  print('Connected.')
+  global DEFAULT_ENGINE
+  global DB_CONNECTION
+  DEFAULT_ENGINE = 'psql'
+  DB_CONNECTION = connection
+
+
 def PostgresJumpStart():
   # Install postgresql server.
   print("Installing and configuring an empty PostgreSQL database.")
@@ -377,12 +441,4 @@ def PostgresJumpStart():
     return
   print('Installation succeeded. Connecting...')
   # Connect to the database.
-  import psycopg2
-  connection = psycopg2.connect(host='localhost', database='logica', user='logica', password='logica')
-  connection.autocommit = True
-
-  print('Connected.')
-  global DEFAULT_ENGINE
-  global DB_CONNECTION
-  DEFAULT_ENGINE = 'psql'
-  DB_CONNECTION = connection
+  ConnectToLocalPostgres()