Skip to content

Commit

Permalink
Don't keep database connection open while running job (#75)
Browse files Browse the repository at this point in the history
  • Loading branch information
@louis-pie
louis-pie authored Jul 15, 2022
1 parent 68cfca2 commit dcf0dcf
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 179 deletions.
91 changes: 3 additions & 88 deletions .pylintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,6 @@ unsafe-load-any-extension=no
# run arbitrary code
extension-pkg-whitelist=ujson

# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality.
optimize-ast=no


[MESSAGES CONTROL]

# Only show warnings with the listed confidence levels. Leave empty to show
Expand Down Expand Up @@ -68,7 +59,6 @@ enable=import-error,
used-before-assignment,
cell-var-from-loop,
global-variable-undefined,
redefine-in-handler,
unused-import,
unused-wildcard-import,
global-variable-not-assigned,
Expand All @@ -77,7 +67,7 @@ enable=import-error,
global-at-module-level,
bad-open-mode,
redundant-unittest-assert,
boolean-datetime
boolean-datetime,
deprecated-method,
anomalous-unicode-escape-in-string,
anomalous-backslash-in-string,
Expand All @@ -101,7 +91,7 @@ enable=import-error,
assert-on-tuple,
dangerous-default-value,
duplicate-key,
useless-else-on-loop
useless-else-on-loop,
expression-not-assigned,
confusing-with-statement,
unnecessary-lambda,
Expand All @@ -113,10 +103,6 @@ enable=import-error,
exec-used,
using-constant-test,
bad-super-call,
missing-super-argument,
slots-on-old-class,
super-on-old-class,
property-on-old-class,
not-an-iterable,
not-a-mapping,
format-needs-mapping,
Expand All @@ -132,36 +118,11 @@ enable=import-error,
bad-format-string,
missing-format-attribute,
missing-format-argument-key,
unused-format-string-argument
unused-format-string-argument,
unused-format-string-key,
invalid-format-index,
bad-indentation,
mixed-indentation,
unnecessary-semicolon,
lowercase-l-suffix,
invalid-encoded-data,
unpacking-in-except,
import-star-module-level,
long-suffix,
old-octal-literal,
old-ne-operator,
backtick,
old-raise-syntax,
metaclass-assignment,
next-method-called,
dict-iter-method,
dict-view-method,
indexing-exception,
raising-string,
using-cmp-argument,
cmp-method,
coerce-method,
delslice-method,
getslice-method,
hex-method,
nonzero-method,
t-method,
setslice-method,
logging-format-truncated,
logging-too-few-args,
logging-too-many-args,
Expand Down Expand Up @@ -206,7 +167,6 @@ enable=import-error,
raising-non-exception,
misplaced-bare-raise,
duplicate-except,
nonstandard-exception,
binary-op-exception,
bare-except,
not-async-context-manager,
Expand Down Expand Up @@ -243,11 +203,6 @@ enable=import-error,
# mypackage.mymodule.MyReporterClass.
output-format=parseable

# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no

# Tells whether to display a full report or only the messages
reports=no

Expand Down Expand Up @@ -282,12 +237,6 @@ ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# else.
single-line-if-stmt=no

# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,dict-separator

# Maximum number of lines in a module
max-module-lines=1000

Expand Down Expand Up @@ -382,10 +331,6 @@ notes=FIXME,XXX


[BASIC]

# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,input

# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_

Expand All @@ -402,63 +347,33 @@ include-naming-hint=no
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,40}$

# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,40}$

# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct constant names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$

# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$

# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$

# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$

# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$

# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$

# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$

# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$

# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,80}$

# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,80}$

# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
Expand Down
171 changes: 87 additions & 84 deletions cicada/commands/exec_schedule.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,109 +153,112 @@ def main(schedule_id, dbname=None):
obj_schedule_details = scheduler.get_schedule_executable(db_cur, schedule_id)

# pylint: disable=too-many-nested-blocks
for row in obj_schedule_details.fetchall():
command = str(row[0])
parameters = str(row[1])
row = obj_schedule_details.fetchone()
command = str(row[0])
parameters = str(row[1])

full_command = []
full_command.extend(command.split())
full_command.extend(parameters.split())
full_command = []
full_command.extend(command.split())
full_command.extend(parameters.split())

human_full_command = str(" ".join(full_command))
human_full_command = str(" ".join(full_command))

# Check to see that schedule is not already running
if get_is_running(db_cur, schedule_id) == 0:
# Initiate schedule log
schedule_log_id = init_schedule_log(
db_cur, server_id, schedule_id, human_full_command
)
reset_adhoc_details(db_cur, schedule_id)
# Check to see that schedule is not already running
if get_is_running(db_cur, schedule_id) == 0:
# Initiate schedule log
schedule_log_id = init_schedule_log(
db_cur, server_id, schedule_id, human_full_command
)
reset_adhoc_details(db_cur, schedule_id)

set_is_running(db_cur, schedule_id)
set_is_running(db_cur, schedule_id)

signal.signal(signal.SIGTERM, catch_sigterm)
signal.signal(signal.SIGQUIT, catch_sigquit)
db_cur.close()
db_conn.close()

cicada_pid = os.getpid()
signal.signal(signal.SIGTERM, catch_sigterm)
signal.signal(signal.SIGQUIT, catch_sigquit)

error_detail = None
returncode = None
cicada_pid = os.getpid()

try:
# pylint: disable=consider-using-with
child_process = subprocess.Popen(
full_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
error_detail = None
returncode = None

# Check if child process has terminated
while returncode is None:
time.sleep(1)
returncode = child_process.poll()

# If still running, check if child_process should be aborted
if returncode is None:
# protect against db unavailable
try:
db_conn = postgres.db_cicada(dbname)
db_cur = db_conn.cursor()
if get_abort_running(db_cur, schedule_id):

# Terminate main process
returncode = -15
error_detail = "Cicada abort_running"
unset_abort_running(db_cur, schedule_id)
db_cur.close()
db_conn.close()
# pylint: disable=unused-variable
except Exception as error:
send_slack_error(
schedule_id,
schedule_log_id,
returncode,
"Cicada database not reachable",
)
time.sleep(1)

# Capture error
except OSError as error:
returncode = error.errno
error_detail = error.strerror
except subprocess.CalledProcessError as error:
returncode = error.returncode
error_detail = "CalledProcessError"
except KeyboardInterrupt:
returncode = 1
error_detail = "KeyboardInterrupt"
except SystemExit:
returncode = 1
error_detail = "SystemExit"
except Exception:
returncode = 999
error_detail = "Crazy Unknown Error"
finally:

# Terminate any zombie processes
for zombie in psutil.Process(cicada_pid).children(recursive=True):
zombie.send_signal(signal.SIGTERM)

# Repeatedly attempt to finalize schedule, even if db is unavailable
db_connection_made = False
while not db_connection_made:
try:
# pylint: disable=consider-using-with
child_process = subprocess.Popen(
full_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)

# Check if child process has terminated
while returncode is None:
time.sleep(1)
returncode = child_process.poll()

# If still running, check if child_process should be aborted
if returncode is None:
# protect against db unavailable
try:
db_conn = postgres.db_cicada(dbname)
db_cur = db_conn.cursor()
db_connection_made = True
except Exception:
if get_abort_running(db_cur, schedule_id):

# Terminate main process
returncode = -15
error_detail = "Cicada abort_running"
unset_abort_running(db_cur, schedule_id)
db_cur.close()
db_conn.close()
# pylint: disable=unused-variable
except Exception as error:
send_slack_error(
schedule_id,
schedule_log_id,
returncode,
"Cicada database not reachable",
"Cicada database not reachable to check abort_running",
)
time.sleep(1)

unset_is_running(db_cur, schedule_id)
finalize_schedule_log(db_cur, schedule_log_id, returncode, error_detail)
# Capture error
except OSError as error:
returncode = error.errno
error_detail = error.strerror
except subprocess.CalledProcessError as error:
returncode = error.returncode
error_detail = "CalledProcessError"
except KeyboardInterrupt:
returncode = 1
error_detail = "KeyboardInterrupt"
except SystemExit:
returncode = 1
error_detail = "SystemExit"
except Exception:
returncode = 999
error_detail = "Crazy Unknown Error"
finally:

# Terminate any zombie processes
for zombie in psutil.Process(cicada_pid).children(recursive=True):
zombie.send_signal(signal.SIGTERM)

# Repeatedly attempt to finalize schedule, even if db is unavailable
db_connection_made = False
while not db_connection_made:
try:
db_conn = postgres.db_cicada(dbname)
db_cur = db_conn.cursor()
db_connection_made = True
except Exception:
send_slack_error(
schedule_id,
schedule_log_id,
returncode,
"Cicada database not reachable to finalize schedule",
)
time.sleep(1)

unset_is_running(db_cur, schedule_id)
finalize_schedule_log(db_cur, schedule_log_id, returncode, error_detail)

db_cur.close()
db_conn.close()
Loading

0 comments on commit dcf0dcf

Please sign in to comment.