Disabled Zemana. Added makefile + tox. lint errors fixed

.gitignore

@@ -6,10 +6,12 @@ _resources
 .env
 .idea/
 .ropeproject
+.tox/
 .vscode/
 *.pyc
 build/
 dist/
 logs/
+report/
 SpamScope.egg-info/
 venv/

.travis.yml

@@ -12,7 +12,6 @@ python:
 env:
     - TIKA_APP_JAR=/tmp/tika-app-${TIKA_VER}.jar
       FAUP_PATH=/tmp/faup
-      ZEMANA_PATH=/tmp/zemana
       DOCKER_ELASTICSEARCH_PATH=/tmp/docker-elasticsearch
 
 before_install:
@@ -48,7 +47,6 @@ install:
     - pip install --upgrade pip setuptools
     - python setup.py install
     - pip install -r requirements_optional.txt
-    - git clone https://$GITHUB_USER:[email protected]/fedelemantuano/zemana-api.git $ZEMANA_PATH && cd $ZEMANA_PATH && python setup.py install && cd -
     - src/cli/faup.sh
     - pip install coveralls
 

Makefile

@@ -0,0 +1,82 @@
+.PHONY: clean clean-test clean-pyc clean-build docs help
+.DEFAULT_GOAL := help
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+
+try:
+	from urllib import pathname2url
+except:
+	from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+
+define PRINT_HELP_PYSCRIPT
+import re, sys
+
+for line in sys.stdin:
+	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
+	if match:
+		target, help = match.groups()
+		print("%-20s %s" % (target, help))
+endef
+export PRINT_HELP_PYSCRIPT
+
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+help:
+	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
+
+clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+
+clean-build: ## remove build artifacts
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc: ## remove Python file artifacts
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## remove test and coverage artifacts
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+	rm -fr .pytest_cache
+
+lint: ## check style with flake8
+	flake8 src tests
+
+test: ## run tests quickly with the default Python
+	python -m unittest discover -s tests -f -v
+
+test-all: ## run tests on every Python version with tox
+	tox
+
+# docs: ## generate Sphinx HTML documentation, including API docs
+# 	rm -f docs/mailparser.rst
+# 	rm -f docs/modules.rst
+# 	sphinx-apidoc -o docs/ mailparser
+# 	$(MAKE) -C docs clean
+# 	$(MAKE) -C docs html
+# 	$(BROWSER) docs/_build/html/index.html
+
+# servedocs: docs ## compile the docs watching for changes
+# 	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
+
+release: dist ## package and upload a release
+	twine upload dist/*
+
+dist: clean ## builds source and wheel package
+	python setup.py sdist
+	python setup.py bdist_wheel
+	ls -l dist
+
+install: clean ## install the package to the active Python's site-packages
+	python setup.py install

requirements-dev.txt

@@ -0,0 +1,28 @@
+# always
+PyYAML
+astropy==1.3.3
+backports.functools-lru-cache>=1.3
+chainmap
+lxml
+mail-parser>=3.4.1
+patool
+pyparsing
+python-magic
+simplejson
+six
+ssdeep
+streamparse==3.13.1
+
+# optional
+elasticsearch>=6.0.0, <7
+redis>=2.10.5, <3
+shodan
+tika-app>=1.4.0
+virustotal-api
+
+# editable
+git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python
+
+# dev
+flake8
+coverage

src/modules/attachments/attachments.py

@@ -141,7 +141,10 @@ def popcontenttype(self, content_type):
         remove = []
 
         for i in self:
-            if not i.get("is_filtered", False):
+            filtered = i.get("is_filtered", False)
+            m_content_type = i["mail_content_type"].lower()
+
+            if not filtered:
                 try:
                     if i["Content-Type"].lower() == content_type:
                         remove.append(i)
@@ -164,8 +167,7 @@ def popcontenttype(self, content_type):
             # you should remove sample from results.
             # You can't use Content-Type because we don't have payload, so
             # we use mail_content_type
-            elif (i.get("is_filtered") and
-                  i["mail_content_type"].lower() == content_type):
+            elif (filtered and m_content_type == content_type):
                 remove.append(i)
 
         else:

src/modules/attachments/post_processing.py

@@ -192,8 +192,8 @@ def thug(conf, attachments):
                         i["thug"] = thug.run(i, **conf)
 
 
-@register(processors, active=True)
-def zemana(conf, attachments):
+@register(processors, active=False)
+def zemana(conf, attachments):  # pragma: no cover
     """This method updates the attachments results
     with Zemana AntiMalware reports.
 

tests/test_attachments.py

@@ -305,12 +305,13 @@ def test_filtercontenttypes(self):
         self.assertEqual(len(t), 1)
         self.assertEqual(len(t[0]["files"]), 0)
 
-    @unittest.skipIf(OPTIONS["THUG_ENABLED"].capitalize() == "False" or
-                     OPTIONS["VIRUSTOTAL_ENABLED"].capitalize() == "False" or
-                     OPTIONS["ZEMANA_ENABLED"].capitalize() == "False",
-                     "Complete post processing test skipped: "
-                     "set env variables 'THUG_ENABLED', "
-                     "'VIRUSTOTAL_ENABLED' and 'ZEMANA_ENABLED' to True")
+    @unittest.skipIf(
+        OPTIONS["THUG_ENABLED"].capitalize() == "False" or OPTIONS[
+            "VIRUSTOTAL_ENABLED"].capitalize() == "False" or OPTIONS[
+                "ZEMANA_ENABLED"].capitalize() == "False",
+        "Complete post processing test skipped: "
+        "set env variables 'THUG_ENABLED', "
+        "'VIRUSTOTAL_ENABLED' and 'ZEMANA_ENABLED' to True")
     def test_post_processing(self):
         t = MailAttachments.withhashes(self.attachments_thug)
         parameters = {
@@ -325,7 +326,7 @@ def test_post_processing(self):
                      "user_agents": ["win7ie90", "winxpie80"],
                      "referer": "http://www.google.com/",
                      "timeout": 300},
-            "zemana": {"enabled": True,
+            "zemana": {"enabled": False,
                        "PartnerId": OPTIONS["ZEMANA_PARTNERID"],
                        "UserId": OPTIONS["ZEMANA_USERID"],
                        "ApiKey": OPTIONS["ZEMANA_APIKEY"],

tests/test_attachments_utils.py

@@ -19,7 +19,6 @@
 
 import logging
 import os
-import sys
 import unittest
 
 import simplejson as json

tests/test_bitmap.py

@@ -18,8 +18,6 @@
 """
 
 import logging
-import os
-import sys
 import unittest
 
 from context import bitmap

tests/test_dialect.py

@@ -18,9 +18,7 @@
 """
 
 
-import datetime
 import logging
-import os
 import unittest
 
 from context import mails

tests/test_network_post_processing.py

@@ -90,11 +90,12 @@ def test_shodan(self):
         shodan(conf, "8.8.8", results)
         self.assertFalse(results)
 
-    @unittest.skipIf(OPTIONS["SHODAN_ENABLED"].capitalize() == "False" or
-                     OPTIONS["VIRUSTOTAL_ENABLED"].capitalize() == "False",
-                     "Complete post processing test skipped: "
-                     "set env variables 'SHODAN_ENABLED' and "
-                     "'VIRUSTOTAL_ENABLED' to True")
+    @unittest.skipIf(
+        OPTIONS["SHODAN_ENABLED"].capitalize() == "False" or OPTIONS[
+            "VIRUSTOTAL_ENABLED"].capitalize() == "False",
+        "Complete post processing test skipped: "
+        "set env variables 'SHODAN_ENABLED' and "
+        "'VIRUSTOTAL_ENABLED' to True")
     def test_processors(self):
         """Test all post processing."""
 

tests/test_phishing.py

@@ -20,7 +20,6 @@
 import logging
 import copy
 import os
-import sys
 import unittest
 
 import mailparser

tox.ini

@@ -0,0 +1,13 @@
+[tox]
+envlist = begin, py27, end
+
+[testenv:begin]
+commands = coverage erase
+
+[testenv]
+deps = -rrequirements-dev.txt
+commands =
+    coverage run --append -m unittest discover -s tests -f -v
+
+[testenv:end]
+commands = coverage html -d {toxinidir}/report