From 0c56c873fa096e503bff67c0b11f098d0a5251f8 Mon Sep 17 00:00:00 2001 From: Henrique Date: Thu, 10 May 2018 14:53:21 -0300 Subject: [PATCH 01/67] init --- requirements_dev.txt | 4 ++++ tests/__init__.py | 0 2 files changed, 4 insertions(+) create mode 100644 requirements_dev.txt create mode 100644 tests/__init__.py diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..5afa89d --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt + +pytest==3.5.1 +pytest-mock==1.10.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 74393c249920f510495771a338fed1a42e05a2fa Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 19:03:43 -0300 Subject: [PATCH 02/67] adding new rules --- .gitignore | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e495ff8..3583576 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,107 @@ -anon.sql +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# dump +*.sql From 731a55eb9267a424c97f2f401700d35009baea60 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 19:05:02 -0300 Subject: [PATCH 03/67] creating simple tasks --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..15de498 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +PIP=.venv/bin/pip +PYTEST=.venv/bin/pytest + +test:clean + PYTHONPATH=. ${PYTEST} -s -v tests/${path} + +venv: + virtualenv .venv + +setup:venv + ${PIP} install -U pip + ${PIP} install -r requirements_dev.txt + +clean: + find . -name "*.pyc" -exec rm -rf {} \; From f0c3f3c29b179fe7ab8bd4c9a12c82289ebd6858 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 19:09:32 -0300 Subject: [PATCH 04/67] pep validation --- anonymize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/anonymize.py b/anonymize.py index 76743da..9b301b9 100755 --- a/anonymize.py +++ b/anonymize.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # This assumes an id on each field. import logging -import hashlib import random @@ -33,6 +32,7 @@ def get_deletes(config): listify = lambda x: x if isinstance(x, list) else [x] + def get_updates(config): global common_hash_secret @@ -64,8 +64,8 @@ def get_updates(config): % dict(field=field)) elif operation == 'hash_email': for field in listify(details): - updates.append("`%(field)s` = CONCAT(MD5(CONCAT(@common_hash_secret, `%(field)s`)), '@mozilla.com')" - % dict(field=field)) + QUERY = "`%(field)s` = CONCAT(MD5(CONCAT(@common_hash_secret, `%(field)s`)), '@mozilla.com')" + updates.append(QUERY % dict(field=field)) elif operation == 'delete': continue else: @@ -79,7 +79,7 @@ def anonymize(config): database = config.get('database', {}) if 'name' in database: - print "USE `%s`;" % database['name'] + print "USE `%s`;" % database['name'] print "SET FOREIGN_KEY_CHECKS=0;" @@ -101,11 +101,11 @@ def anonymize(config): if len(sys.argv) > 1: files = sys.argv[1:] else: - files = [ 'anonymize.yml' ] + files = ['anonymize.yml', ] for f in files: print "--" - print "-- %s" %f + print "-- %s" % f print "--" print "SET @common_hash_secret=rand();" print "" From eff9c9037d2e61a6a4cb9a1ed86603c73d919c06 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 19:58:22 -0300 Subject: [PATCH 05/67] starting refactoring --- Makefile | 2 +- anonymize/__init__.py | 11 +++++++++++ anonymize.py => anonymize/anonymize.py | 1 - anonymize.yml => anonymize/anonymize.yml | 0 .../developer_mozilla_org.yml | 0 5 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 anonymize/__init__.py rename anonymize.py => anonymize/anonymize.py (99%) rename anonymize.yml => anonymize/anonymize.yml (100%) rename developer_mozilla_org.yml => anonymize/developer_mozilla_org.yml (100%) diff --git a/Makefile b/Makefile index 15de498..4b48e0d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PIP=.venv/bin/pip PYTEST=.venv/bin/pytest test:clean - PYTHONPATH=. ${PYTEST} -s -v tests/${path} + PYTHONPATH=anonymize ${PYTEST} -s -v tests/${path} venv: virtualenv .venv diff --git a/anonymize/__init__.py b/anonymize/__init__.py new file mode 100644 index 0000000..1fc2ac3 --- /dev/null +++ b/anonymize/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from optparse import OptionParser + + +def main(): + parser = OptionParser() + parser.add_option('-y', '--yaml') + +if __name__ == "__main__": + main() diff --git a/anonymize.py b/anonymize/anonymize.py similarity index 99% rename from anonymize.py rename to anonymize/anonymize.py index 9b301b9..acaaa0c 100755 --- a/anonymize.py +++ b/anonymize/anonymize.py @@ -77,7 +77,6 @@ def get_updates(config): def anonymize(config): database = config.get('database', {}) - if 'name' in database: print "USE `%s`;" % database['name'] diff --git a/anonymize.yml b/anonymize/anonymize.yml similarity index 100% rename from anonymize.yml rename to anonymize/anonymize.yml diff --git a/developer_mozilla_org.yml b/anonymize/developer_mozilla_org.yml similarity index 100% rename from developer_mozilla_org.yml rename to anonymize/developer_mozilla_org.yml From 13adecc7b124a3423a763e96af1a72b4d23e183d Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 20:09:11 -0300 Subject: [PATCH 06/67] creating the first test for get_truncates --- anonymize/anonymize.py | 2 +- tests/test_truncates.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/test_truncates.py diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index acaaa0c..2c2d5d4 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -13,7 +13,7 @@ def get_truncates(config): truncates = database.get('truncate', []) sql = [] for truncate in truncates: - sql.append('TRUNCATE `%s`' % truncate) + sql.append('TRUNCATE `{}`'.format(truncate)) return sql diff --git a/tests/test_truncates.py b/tests/test_truncates.py new file mode 100644 index 0000000..866e084 --- /dev/null +++ b/tests/test_truncates.py @@ -0,0 +1,20 @@ +from anonymize.anonymize import get_truncates + + +def test_should_get_the_empty_list(): + truncates = get_truncates({}) + + assert truncates == [] + + +def test_should_get_the_list_of_truncate_tables(): + truncates = get_truncates({ + "database": { + "truncate": [ + "user", + "subscribers" + ] + } + }) + + assert truncates == ['TRUNCATE `user`', 'TRUNCATE `subscribers`'] From 6875784a3894dd9f5aaf8e861a5474f19748f355 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 20:14:24 -0300 Subject: [PATCH 07/67] fixing a bug --- anonymize/anonymize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index 2c2d5d4..760201a 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -19,7 +19,8 @@ def get_truncates(config): def get_deletes(config): database = config.get('database', {}) - tables = database.get('tables', []) + tables = database.get('tables', {}) + sql = [] for table, data in tables.iteritems(): if 'delete' in data: From fe1a693b317b500dc86b802040d565a6cfeec1ba Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 20:14:39 -0300 Subject: [PATCH 08/67] fixing a bug --- tests/test_get_deletes.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/test_get_deletes.py diff --git a/tests/test_get_deletes.py b/tests/test_get_deletes.py new file mode 100644 index 0000000..d97bbeb --- /dev/null +++ b/tests/test_get_deletes.py @@ -0,0 +1,7 @@ +from anonymize.anonymize import get_deletes + + +def test_should_get_the_empty_list(): + truncates = get_deletes({}) + + assert truncates == [] From 97d4bdbc0cd4036fd644ab618b3a09aaf1d9d33d Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 20:24:55 -0300 Subject: [PATCH 09/67] writing a new test for sql delete --- tests/test_get_deletes.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_get_deletes.py b/tests/test_get_deletes.py index d97bbeb..a2a6dd1 100644 --- a/tests/test_get_deletes.py +++ b/tests/test_get_deletes.py @@ -5,3 +5,17 @@ def test_should_get_the_empty_list(): truncates = get_deletes({}) assert truncates == [] + + +def test_should_get_the_list_of_delete_itens(): + deleted = get_deletes({ + "database": { + "tables": { + "user": { + "delete": {"id": 1} + } + } + } + }) + + assert deleted == ['DELETE FROM `user` WHERE `id` = "1"'] From e8c6c91e56166f890cf0be5489472113d4fa4908 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 22:31:37 -0300 Subject: [PATCH 10/67] covering def listify --- tests/test_listify.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/test_listify.py diff --git a/tests/test_listify.py b/tests/test_listify.py new file mode 100644 index 0000000..9f168d1 --- /dev/null +++ b/tests/test_listify.py @@ -0,0 +1,9 @@ +from anonymize.anonymize import listify + + +def test_should_get_a_list_item(): + assert listify("") == [""] + + +def test_should_get_a_list_none(): + assert listify(None) == [None, ] From 7c0dc6e495eace04b09d8fc640976c01514703e2 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 22:44:41 -0300 Subject: [PATCH 11/67] updating the default domain and creating a simple test --- anonymize/anonymize.py | 4 ++-- tests/test_get_updates.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 tests/test_get_updates.py diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index 760201a..48afc1c 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -54,7 +54,7 @@ def get_updates(config): updates.append("`%s` = INET_NTOA(RAND()*1000000000)" % field) elif operation == 'random_email': for field in listify(details): - updates.append("`%s` = CONCAT(id, '@mozilla.com')" + updates.append("`%s` = CONCAT(id, '@example.com')" % field) elif operation == 'random_username': for field in listify(details): @@ -65,7 +65,7 @@ def get_updates(config): % dict(field=field)) elif operation == 'hash_email': for field in listify(details): - QUERY = "`%(field)s` = CONCAT(MD5(CONCAT(@common_hash_secret, `%(field)s`)), '@mozilla.com')" + QUERY = "`%(field)s` = CONCAT(MD5(CONCAT(@common_hash_secret, `%(field)s`)), '@example.com')" updates.append(QUERY % dict(field=field)) elif operation == 'delete': continue diff --git a/tests/test_get_updates.py b/tests/test_get_updates.py new file mode 100644 index 0000000..fb9330d --- /dev/null +++ b/tests/test_get_updates.py @@ -0,0 +1,18 @@ +from anonymize.anonymize import get_updates + + +def test_should_get_the_update_list(): + data = get_updates({ + "database": { + "tables": { + "user": { + "nullify": ["phone", ], + "random_email": ["email", ], + "random_ip": ['ip'] + } + } + } + }) + + r = ["UPDATE `user` SET `phone` = NULL, `ip` = INET_NTOA(RAND()*1000000000), `email` = CONCAT(id, '@example.com')"] + assert data == r From 32a642c6449ac1dbdf86ea37afd83876e65a14f9 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 10 May 2018 22:55:10 -0300 Subject: [PATCH 12/67] adding a comment --- anonymize/anonymize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index 48afc1c..a4df65d 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -43,6 +43,8 @@ def get_updates(config): for table, data in tables.iteritems(): updates = [] for operation, details in data.iteritems(): + + # tables columns if operation == 'nullify': for field in listify(details): updates.append("`%s` = NULL" % field) @@ -54,8 +56,7 @@ def get_updates(config): updates.append("`%s` = INET_NTOA(RAND()*1000000000)" % field) elif operation == 'random_email': for field in listify(details): - updates.append("`%s` = CONCAT(id, '@example.com')" - % field) + updates.append("`%s` = CONCAT(id, '@example.com')" % field) elif operation == 'random_username': for field in listify(details): updates.append("`%s` = CONCAT('_user_', id)" % field) From adcf6da9fb5d87d74fd942160acdf6e194c0a9b9 Mon Sep 17 00:00:00 2001 From: Henrique Date: Fri, 11 May 2018 08:01:23 -0300 Subject: [PATCH 13/67] Refactoring * creating a setup for project. * renaming samples files. * simple example of option parse. --- anonymize/__init__.py | 10 +++++ anonymize/{anonymize.yml => sample.yml} | 0 ...{developer_mozilla_org.yml => sample2.yml} | 0 setup.py | 42 +++++++++++++++++++ 4 files changed, 52 insertions(+) rename anonymize/{anonymize.yml => sample.yml} (100%) rename anonymize/{developer_mozilla_org.yml => sample2.yml} (100%) create mode 100644 setup.py diff --git a/anonymize/__init__.py b/anonymize/__init__.py index 1fc2ac3..fc4b551 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -3,9 +3,19 @@ from optparse import OptionParser +class Anonymize(object): + pass + + def main(): parser = OptionParser() parser.add_option('-y', '--yaml') + parser.add_option('-s', '--sample', default=1) + (options, args) = parser.parse_args() + + anonymize = Anonymize() + anonymize.run() + if __name__ == "__main__": main() diff --git a/anonymize/anonymize.yml b/anonymize/sample.yml similarity index 100% rename from anonymize/anonymize.yml rename to anonymize/sample.yml diff --git a/anonymize/developer_mozilla_org.yml b/anonymize/sample2.yml similarity index 100% rename from anonymize/developer_mozilla_org.yml rename to anonymize/sample2.yml diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2841c40 --- /dev/null +++ b/setup.py @@ -0,0 +1,42 @@ +""" + Contributors can benefit from having real data when they are developing. This script can do a few things (see anonymize.yml): + + Truncate any tables (logs, and other cruft which may have sensitive data) + + Nullify fields (emails, passwords, etc) + + Fill in random/arbitrary data: + + Random integers + Random IP addresses + Email addresses + Usernames + Delete rows based on simple rules: e.g. DELETE FROM mytable WHERE private = "Yes": + + database: tables: mytable: delete: private: Yes +""" + +from setuptools import setup + +setup_params = { + "entry_points": { + "console_scripts": [ + "anonymize=anonymize:main" + ] + } +} + + +setup( + author="Dave Dash", + author_email="dd+github@davedash.com, contato@henriquelopes.com.br", + version='0.2', + name="Mysql Anonymous", + url="https://github.com/davedash/mysql-anonymous", + packages=["anonymize"], + platforms=['python >= 2.7'], + description=__doc__, + long_description=__doc__, + py_modules=["anonymize"], + **setup_params +) From 15ac47577be237a2c13873a0672f2f29b983424f Mon Sep 17 00:00:00 2001 From: Henrique Date: Fri, 11 May 2018 08:08:54 -0300 Subject: [PATCH 14/67] adding ci support --- .travis.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..cf4329f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +language: python + +python: + - "2.7" + - "3.5" + - "3.6" + +install: "pip install -r requirements_dev.txt" + +script: + - make test From ca93a06fbd13353858a0b4fba6526d7f73951019 Mon Sep 17 00:00:00 2001 From: Henrique Date: Fri, 11 May 2018 08:09:55 -0300 Subject: [PATCH 15/67] adding the status ci image --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 05661c9..e57490b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ## Mysql Anonymous +[![Build Status](https://travis-ci.org/riquellopes/mysql-anonymous.svg?branch=master)](https://travis-ci.org/riquellopes/mysql-anonymous) + Contributors can benefit from having real data when they are developing. This script can do a few things (see `anonymize.yml`): From 21775a7f7c360d7e2d6829a02d1c95a4a32d3eb1 Mon Sep 17 00:00:00 2001 From: Henrique Date: Fri, 11 May 2018 08:11:31 -0300 Subject: [PATCH 16/67] fix the install command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cf4329f..4cde52b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - "3.5" - "3.6" -install: "pip install -r requirements_dev.txt" +install: "make setup" script: - make test From 97a777982e8150e4ed389f3a6e39cb319df80d40 Mon Sep 17 00:00:00 2001 From: Henrique Date: Fri, 11 May 2018 10:27:10 -0300 Subject: [PATCH 17/67] changing import package at top --- anonymize/anonymize.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index a4df65d..f72d097 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -2,6 +2,8 @@ # This assumes an id on each field. import logging import random +import yaml +import sys log = logging.getLogger('anonymize') @@ -95,10 +97,6 @@ def anonymize(config): print if __name__ == '__main__': - - import yaml - import sys - if len(sys.argv) > 1: files = sys.argv[1:] else: From f42a92a18ebda6dd0fdb3a5baa59ac4c2f4235ab Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Fri, 11 May 2018 18:23:05 -0300 Subject: [PATCH 18/67] refactoring --- anonymize/__init__.py | 11 ++++++++++- anonymize/anonymize.py | 26 +++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/anonymize/__init__.py b/anonymize/__init__.py index fc4b551..c0ada52 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -1,10 +1,19 @@ #!/usr/bin/env python from __future__ import absolute_import +import yaml from optparse import OptionParser +from anonymize import AnonymizeSchemes class Anonymize(object): - pass + + def __init__(self, file_name="sample.yml"): + self._file_name = file_name + + def run(self): + with open(self._file_name) as handle: + a = AnonymizeSchemes(yaml.load(handle)) + a.build() def main(): diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index f72d097..65be84f 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -100,7 +100,7 @@ def anonymize(config): if len(sys.argv) > 1: files = sys.argv[1:] else: - files = ['anonymize.yml', ] + files = ['sample.yml', ] for f in files: print "--" @@ -116,3 +116,27 @@ def anonymize(config): for name, sub_cfg in databases.items(): print "USE `%s`;" % name anonymize({'database': sub_cfg}) + + +class AnonymizeScheme(object): + pass + + +class AnonymizeSchemes(object): + + def __init__(self, cfg): + self._cfg = cfg + self._print_use = False + + def build(self): + for name, cfg in self._databases().items(): + if self._print_use: + print "USE `{}`;".format(name) + a = AnonymizeScheme(name, cfg) + a.create() + + def _databases(self): + if "databases" in self._cfg: + self._print_use = True + return self._cfg.get("databases") + return {"default": self._cfg.get("database")} From 1093e9b31d323d8fb284e1b7413df9983e4fed1b Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Fri, 11 May 2018 20:33:21 -0300 Subject: [PATCH 19/67] refactoring --- Makefile | 4 + anonymize/__init__.py | 23 ++- anonymize/anonymize.py | 204 +++++++++++++------------- anonymize/{sample.yml => sample1.yml} | 0 tests/test_get_deletes.py | 16 +- tests/test_get_updates.py | 18 +-- tests/test_truncates.py | 18 +-- 7 files changed, 144 insertions(+), 139 deletions(-) rename anonymize/{sample.yml => sample1.yml} (100%) diff --git a/Makefile b/Makefile index 4b48e0d..ac20885 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ PIP=.venv/bin/pip PYTEST=.venv/bin/pytest +PYTHON=.venv/bin/python test:clean PYTHONPATH=anonymize ${PYTEST} -s -v tests/${path} @@ -13,3 +14,6 @@ setup:venv clean: find . -name "*.pyc" -exec rm -rf {} \; + +sample: + PYTHONPATH=anonymize ${PYTHON} anonymize/__init__.py diff --git a/anonymize/__init__.py b/anonymize/__init__.py index c0ada52..581ebf0 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import absolute_import +import os import yaml from optparse import OptionParser from anonymize import AnonymizeSchemes @@ -7,22 +7,35 @@ class Anonymize(object): - def __init__(self, file_name="sample.yml"): + def __init__(self, file_name, sample): self._file_name = file_name + self._sample = sample def run(self): - with open(self._file_name) as handle: + with open(self.file_name) as handle: a = AnonymizeSchemes(yaml.load(handle)) a.build() + @property + def file_name(self): + if self._sample: + BASE_DIR = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(__file__)))) + return os.path.join( + BASE_DIR, 'anonymize', "sample{}.yml".format(self._sample)) + return self._file_name + def main(): parser = OptionParser() parser.add_option('-y', '--yaml') parser.add_option('-s', '--sample', default=1) - (options, args) = parser.parse_args() + (options, __) = parser.parse_args() - anonymize = Anonymize() + anonymize = Anonymize(**{ + "file_name": options.yaml, + "sample": options.sample + }) anonymize.run() diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index 65be84f..93b3183 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -1,125 +1,115 @@ #!/usr/bin/env python -# This assumes an id on each field. +from __future__ import print_function +import itertools import logging import random -import yaml -import sys log = logging.getLogger('anonymize') common_hash_secret = "%016x" % (random.getrandbits(128)) +listify = lambda x: x if isinstance(x, list) else [x] -def get_truncates(config): - database = config.get('database', {}) - truncates = database.get('truncate', []) - sql = [] - for truncate in truncates: - sql.append('TRUNCATE `{}`'.format(truncate)) - return sql +class AnonymizeBaseAction(list): -def get_deletes(config): - database = config.get('database', {}) - tables = database.get('tables', {}) + def __init__(self, scheme): + self._scheme = scheme + self.create() - sql = [] - for table, data in tables.iteritems(): - if 'delete' in data: - fields = [] - for f, v in data['delete'].iteritems(): - fields.append('`%s` = "%s"' % (f, v)) - statement = 'DELETE FROM `%s` WHERE ' % table + ' AND '.join(fields) - sql.append(statement) - return sql -listify = lambda x: x if isinstance(x, list) else [x] +class AnonymizeTruncate(AnonymizeBaseAction): + + def create(self): + for truncate in self._scheme.database.get("truncate", []): + self.append('TRUNCATE `{}`'.format(truncate)) + + +class AnonymizeDelete(AnonymizeBaseAction): + def create(self): + for table, data in self._scheme.tables.iteritems(): + if 'delete' in data: + self.append('DELETE FROM `{}` WHERE '.format(table) + ' AND '.join( + ['`{}` = "{}"'.format(f, v) for f, v in data['delete'].iteritems()] + )) -def get_updates(config): - global common_hash_secret - - database = config.get('database', {}) - tables = database.get('tables', []) - sql = [] - for table, data in tables.iteritems(): - updates = [] - for operation, details in data.iteritems(): - - # tables columns - if operation == 'nullify': - for field in listify(details): - updates.append("`%s` = NULL" % field) - elif operation == 'random_int': - for field in listify(details): - updates.append("`%s` = ROUND(RAND()*1000000)" % field) - elif operation == 'random_ip': - for field in listify(details): - updates.append("`%s` = INET_NTOA(RAND()*1000000000)" % field) - elif operation == 'random_email': - for field in listify(details): - updates.append("`%s` = CONCAT(id, '@example.com')" % field) - elif operation == 'random_username': - for field in listify(details): - updates.append("`%s` = CONCAT('_user_', id)" % field) - elif operation == 'hash_value': - for field in listify(details): - updates.append("`%(field)s` = MD5(CONCAT(@common_hash_secret, `%(field)s`))" - % dict(field=field)) - elif operation == 'hash_email': - for field in listify(details): - QUERY = "`%(field)s` = CONCAT(MD5(CONCAT(@common_hash_secret, `%(field)s`)), '@example.com')" - updates.append(QUERY % dict(field=field)) - elif operation == 'delete': - continue - else: - log.warning('Unknown operation.') - if updates: - sql.append('UPDATE `%s` SET %s' % (table, ', '.join(updates))) - return sql - - -def anonymize(config): - database = config.get('database', {}) - if 'name' in database: - print "USE `%s`;" % database['name'] - - print "SET FOREIGN_KEY_CHECKS=0;" - - sql = [] - sql.extend(get_truncates(config)) - sql.extend(get_deletes(config)) - sql.extend(get_updates(config)) - for stmt in sql: - print stmt + ';' - - print "SET FOREIGN_KEY_CHECKS=1;" - print - -if __name__ == '__main__': - if len(sys.argv) > 1: - files = sys.argv[1:] - else: - files = ['sample.yml', ] - - for f in files: - print "--" - print "-- %s" % f - print "--" - print "SET @common_hash_secret=rand();" - print "" - cfg = yaml.load(open(f)) - if 'databases' not in cfg: - anonymize(cfg) - else: - databases = cfg.get('databases') - for name, sub_cfg in databases.items(): - print "USE `%s`;" % name - anonymize({'database': sub_cfg}) + +class AnonymizeUpdate(AnonymizeBaseAction): + + def create(self): + global common_hash_secret + + for table, data in self._scheme.tables.iteritems(): + updates = [] + for operation, details in data.iteritems(): + + # tables columns + if operation == 'nullify': + for field in listify(details): + updates.append("`%s` = NULL" % field) + elif operation == 'random_int': + for field in listify(details): + updates.append("`%s` = ROUND(RAND()*1000000)" % field) + elif operation == 'random_ip': + for field in listify(details): + updates.append("`%s` = INET_NTOA(RAND()*1000000000)" % field) + elif operation == 'random_email': + for field in listify(details): + updates.append("`%s` = CONCAT(id, '@example.com')" % field) + elif operation == 'random_username': + for field in listify(details): + updates.append("`%s` = CONCAT('_user_', id)" % field) + elif operation == 'hash_value': + for field in listify(details): + updates.append("`%(field)s` = MD5(CONCAT(@common_hash_secret, `%(field)s`))" + % dict(field=field)) + elif operation == 'hash_email': + for field in listify(details): + QUERY = "`%(field)s` = CONCAT(MD5(CONCAT(@common_hash_secret, `%(field)s`)), '@example.com')" + updates.append(QUERY % dict(field=field)) + elif operation == 'delete': + continue + else: + log.warning('Unknown operation.') + if updates: + self.append('UPDATE `%s` SET %s' % (table, ', '.join(updates))) class AnonymizeScheme(object): - pass + + def __init__(self, name, cfg): + self._name = name + self._cfg = cfg + + def create(self): + if self._print_use(): + print("USE `{}`".format(self.name)) + print("SET FOREIGN_KEY_CHECKS=0;") + + for action in self._actions(): + print("{};".format(action)) + print("SET FOREIGN_KEY_CHECKS=1;") + print() + + @property + def database(self): + return self._cfg + + @property + def tables(self): + return self.database.get("tables", {}) + + def _print_use(self): + return "name" in self.database + + @property + def name(self): + return self.database['name'] or self._name + + def _actions(self): + return itertools.chain( + AnonymizeTruncate(self), AnonymizeDelete(self), AnonymizeUpdate(self)) class AnonymizeSchemes(object): @@ -129,9 +119,13 @@ def __init__(self, cfg): self._print_use = False def build(self): + print("--") + print("SET @common_hash_secret=rand();") + print("") + for name, cfg in self._databases().items(): if self._print_use: - print "USE `{}`;".format(name) + print("USE `{}`;".format(name)) a = AnonymizeScheme(name, cfg) a.create() diff --git a/anonymize/sample.yml b/anonymize/sample1.yml similarity index 100% rename from anonymize/sample.yml rename to anonymize/sample1.yml diff --git a/tests/test_get_deletes.py b/tests/test_get_deletes.py index a2a6dd1..fc0df2a 100644 --- a/tests/test_get_deletes.py +++ b/tests/test_get_deletes.py @@ -1,21 +1,19 @@ -from anonymize.anonymize import get_deletes +from anonymize.anonymize import AnonymizeDelete, AnonymizeScheme def test_should_get_the_empty_list(): - truncates = get_deletes({}) + truncates = AnonymizeDelete(AnonymizeScheme("default", {})) assert truncates == [] def test_should_get_the_list_of_delete_itens(): - deleted = get_deletes({ - "database": { - "tables": { - "user": { - "delete": {"id": 1} - } + deleted = AnonymizeDelete(AnonymizeScheme("default", { + "tables": { + "user": { + "delete": {"id": 1} } } - }) + })) assert deleted == ['DELETE FROM `user` WHERE `id` = "1"'] diff --git a/tests/test_get_updates.py b/tests/test_get_updates.py index fb9330d..417221b 100644 --- a/tests/test_get_updates.py +++ b/tests/test_get_updates.py @@ -1,18 +1,16 @@ -from anonymize.anonymize import get_updates +from anonymize.anonymize import AnonymizeUpdate, AnonymizeScheme def test_should_get_the_update_list(): - data = get_updates({ - "database": { - "tables": { - "user": { - "nullify": ["phone", ], - "random_email": ["email", ], - "random_ip": ['ip'] - } + data = AnonymizeUpdate(AnonymizeScheme("default", { + "tables": { + "user": { + "nullify": ["phone", ], + "random_email": ["email", ], + "random_ip": ['ip'] } } - }) + })) r = ["UPDATE `user` SET `phone` = NULL, `ip` = INET_NTOA(RAND()*1000000000), `email` = CONCAT(id, '@example.com')"] assert data == r diff --git a/tests/test_truncates.py b/tests/test_truncates.py index 866e084..c4f3739 100644 --- a/tests/test_truncates.py +++ b/tests/test_truncates.py @@ -1,20 +1,18 @@ -from anonymize.anonymize import get_truncates +from anonymize.anonymize import AnonymizeTruncate, AnonymizeScheme def test_should_get_the_empty_list(): - truncates = get_truncates({}) + truncates = AnonymizeTruncate(AnonymizeScheme("default", {})) assert truncates == [] def test_should_get_the_list_of_truncate_tables(): - truncates = get_truncates({ - "database": { - "truncate": [ - "user", - "subscribers" - ] - } - }) + truncates = AnonymizeTruncate(AnonymizeScheme("default", { + "truncate": [ + "user", + "subscribers" + ] + })) assert truncates == ['TRUNCATE `user`', 'TRUNCATE `subscribers`'] From 1ffc9fbc82f9db8a7faf7ea108e1aa611122bea6 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Fri, 11 May 2018 21:28:17 -0300 Subject: [PATCH 20/67] adding support to dynamic primary key --- anonymize/anonymize.py | 6 ++++-- anonymize/sample1.yml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index 93b3183..11cd948 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -42,6 +42,8 @@ def create(self): for table, data in self._scheme.tables.iteritems(): updates = [] + primary_key = data.pop('primary_key', "id") + for operation, details in data.iteritems(): # tables columns @@ -56,10 +58,10 @@ def create(self): updates.append("`%s` = INET_NTOA(RAND()*1000000000)" % field) elif operation == 'random_email': for field in listify(details): - updates.append("`%s` = CONCAT(id, '@example.com')" % field) + updates.append("`{}` = CONCAT({}, '@example.com')".format(field, primary_key)) elif operation == 'random_username': for field in listify(details): - updates.append("`%s` = CONCAT('_user_', id)" % field) + updates.append("`{}` = CONCAT('_user_', {})".format(field, primary_key)) elif operation == 'hash_value': for field in listify(details): updates.append("`%(field)s` = MD5(CONCAT(@common_hash_secret, `%(field)s`))" diff --git a/anonymize/sample1.yml b/anonymize/sample1.yml index e078d17..b969a06 100644 --- a/anonymize/sample1.yml +++ b/anonymize/sample1.yml @@ -52,6 +52,7 @@ database: - versioncomments tables: addons: + primary_key: 1 nullify: [nominationmessage, paypal_id, charity_id] random_int: - average_daily_downloads From e29e98933c0e911c8832f39cce94e042d6aff771 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 19:11:07 -0300 Subject: [PATCH 21/67] setting the log on project --- anonymize/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/anonymize/__init__.py b/anonymize/__init__.py index 581ebf0..433c6d5 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -1,9 +1,15 @@ #!/usr/bin/env python import os import yaml +import logging from optparse import OptionParser from anonymize import AnonymizeSchemes +logging.basicConfig( + filename='anonymize.log', + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + level=logging.DEBUG) + class Anonymize(object): From 85f29ffceb8b7b2fe006a654ce9d684632ba4851 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 19:15:56 -0300 Subject: [PATCH 22/67] refactoring code to improve the tests --- anonymize/field.py | 80 +++++++++++++++++++++++++++++++++++++ tests/test_field_methods.py | 31 ++++++++++++++ tests/test_listify.py | 9 ----- 3 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 anonymize/field.py create mode 100644 tests/test_field_methods.py delete mode 100644 tests/test_listify.py diff --git a/anonymize/field.py b/anonymize/field.py new file mode 100644 index 0000000..4507135 --- /dev/null +++ b/anonymize/field.py @@ -0,0 +1,80 @@ +import logging +logger = logging.getLogger('anonymize') + + +class Field(object): + + def __init__(self, field, primary_key="id"): + self._field = field + self._primary_key = primary_key + + def render(self): + return self.sql_field.format( + field=self._field, primary_key=self._primary_key) + + +class Nullify(Field): + sql_field = "`{field}` = NULL" + + +class RandomInt(Field): + sql_field = "`{field}` = ROUND(RAND()*1000000)" + + +class RandomIp(Field): + sql_field = "`{field}` = INET_NTOA(RAND()*1000000000)" + + +class RandomEmail(Field): + sql_field = "`{field}` = CONCAT({primary_key}, '@example.com')" + + +class RandomUsername(Field): + sql_field = "`{field}` = CONCAT('_user_', {primary_key})" + + +class HashValue(Field): + sql_field = "`{field}` = MD5(CONCAT(@common_hash_secret, `{field}`))" + + +class HashEmail(Field): + sql_field = "`{field}` = CONCAT(MD5(CONCAT(@common_hash_secret, `{field}`)), '@example.com')" + + +class AnonymizeField(object): + + def __init__(self, data, primary_key): + self._data = data + self._primary_key = primary_key + + self._fields = { + "nullify": Nullify, + "random_int": RandomInt, + "random_ip": RandomIp, + "random_email": RandomEmail, + "random_username": RandomUsername, + "hash_value": HashValue, + "hash_email": HashEmail, + } + + def build(self): + for operation, details in self._data.iteritems(): + if self._valid_operation(operation): + for field in self._listify(details): + yield self.get_field(operation, field) + else: + logger.warning("Unknown {} operation.".format(operation)) + + def _valid_operation(self, operation): + return operation in self._fields + + def _delete_operation(self, operation): + return operation == "delete" + + def _listify(self, values): + if isinstance(values, list): + return values + return [values, ] + + def get_field(self, operation, field): + return self._fields[operation](field, self._primary_key) diff --git a/tests/test_field_methods.py b/tests/test_field_methods.py new file mode 100644 index 0000000..0e3a99e --- /dev/null +++ b/tests/test_field_methods.py @@ -0,0 +1,31 @@ +import pytest +from anonymize.field import AnonymizeField, Nullify + + +@pytest.fixture +def anon(): + return AnonymizeField({}, "id") + + +def test_should_get_a_list_item(anon): + assert anon._listify("") == [""] + + +def test_should_get_a_list_none(anon): + assert anon._listify(None) == [None, ] + + +def test_should_get_true(anon): + assert anon._valid_operation("nullify") + + +def test_should_get_false(anon): + assert anon._valid_operation("nullifys") is False + + +def test_should_get_true_for_delete_operations(anon): + assert anon._delete_operation("delete") + + +def test_should_get_Nullify_instance(anon): + assert isinstance(anon.get_field("nullify", "id"), Nullify) diff --git a/tests/test_listify.py b/tests/test_listify.py deleted file mode 100644 index 9f168d1..0000000 --- a/tests/test_listify.py +++ /dev/null @@ -1,9 +0,0 @@ -from anonymize.anonymize import listify - - -def test_should_get_a_list_item(): - assert listify("") == [""] - - -def test_should_get_a_list_none(): - assert listify(None) == [None, ] From ec3d17d12e36ef9261c090e4065b431d9d7fdee6 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 19:18:42 -0300 Subject: [PATCH 23/67] breaking de code into small parts --- anonymize/anonymize.py | 44 ++++++------------------------------------ 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index 11cd948..f7aeff4 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -1,15 +1,10 @@ -#!/usr/bin/env python from __future__ import print_function import itertools -import logging import random +from field import AnonymizeField - -log = logging.getLogger('anonymize') common_hash_secret = "%016x" % (random.getrandbits(128)) -listify = lambda x: x if isinstance(x, list) else [x] - class AnonymizeBaseAction(list): @@ -42,38 +37,11 @@ def create(self): for table, data in self._scheme.tables.iteritems(): updates = [] - primary_key = data.pop('primary_key', "id") - - for operation, details in data.iteritems(): - - # tables columns - if operation == 'nullify': - for field in listify(details): - updates.append("`%s` = NULL" % field) - elif operation == 'random_int': - for field in listify(details): - updates.append("`%s` = ROUND(RAND()*1000000)" % field) - elif operation == 'random_ip': - for field in listify(details): - updates.append("`%s` = INET_NTOA(RAND()*1000000000)" % field) - elif operation == 'random_email': - for field in listify(details): - updates.append("`{}` = CONCAT({}, '@example.com')".format(field, primary_key)) - elif operation == 'random_username': - for field in listify(details): - updates.append("`{}` = CONCAT('_user_', {})".format(field, primary_key)) - elif operation == 'hash_value': - for field in listify(details): - updates.append("`%(field)s` = MD5(CONCAT(@common_hash_secret, `%(field)s`))" - % dict(field=field)) - elif operation == 'hash_email': - for field in listify(details): - QUERY = "`%(field)s` = CONCAT(MD5(CONCAT(@common_hash_secret, `%(field)s`)), '@example.com')" - updates.append(QUERY % dict(field=field)) - elif operation == 'delete': - continue - else: - log.warning('Unknown operation.') + anon = AnonymizeField(data, data.pop('primary_key', "id")) + + for n in anon.build(): + updates.append(n.render()) + if updates: self.append('UPDATE `%s` SET %s' % (table, ', '.join(updates))) From 0a86f306bf60f6778d4a731313b8498e953e539e Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 19:40:49 -0300 Subject: [PATCH 24/67] adding new random fields --- README.md | 3 +++ anonymize/field.py | 15 +++++++++++++++ tests/test_field_methods.py | 21 ++++++++++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e57490b..b136875 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,9 @@ developing. This script can do a few things (see `anonymize.yml`): * Fill in random/arbitrary data: * Random integers * Random IP addresses + * Random Cell Phone + * Random Phone + * Random [CPF](https://pt.wikipedia.org/wiki/Cadastro_de_pessoas_f%C3%ADsicas) * Email addresses * Usernames * Delete rows based on simple rules: e.g. diff --git a/anonymize/field.py b/anonymize/field.py index 4507135..d23aef3 100644 --- a/anonymize/field.py +++ b/anonymize/field.py @@ -33,6 +33,18 @@ class RandomUsername(Field): sql_field = "`{field}` = CONCAT('_user_', {primary_key})" +class RandomCellPhone(Field): + sql_field = "`{field}` = LPAD({primary_key}, 13, 5)" + + +class RandomPhone(Field): + sql_field = "`{field}` = LPAD({primary_key}, 12, 5)" + + +class RandomCpf(Field): + sql_field = "`{field}` = LPAD({primary_key}, 11, 5)" + + class HashValue(Field): sql_field = "`{field}` = MD5(CONCAT(@common_hash_secret, `{field}`))" @@ -53,6 +65,9 @@ def __init__(self, data, primary_key): "random_ip": RandomIp, "random_email": RandomEmail, "random_username": RandomUsername, + "random_cell_phone": RandomCellPhone, + "random_phone": RandomPhone, + "random_cpf": RandomCpf, "hash_value": HashValue, "hash_email": HashEmail, } diff --git a/tests/test_field_methods.py b/tests/test_field_methods.py index 0e3a99e..1361901 100644 --- a/tests/test_field_methods.py +++ b/tests/test_field_methods.py @@ -1,5 +1,6 @@ import pytest -from anonymize.field import AnonymizeField, Nullify +from anonymize.field import ( + AnonymizeField, Nullify, RandomCellPhone, RandomPhone, RandomCpf) @pytest.fixture @@ -29,3 +30,21 @@ def test_should_get_true_for_delete_operations(anon): def test_should_get_Nullify_instance(anon): assert isinstance(anon.get_field("nullify", "id"), Nullify) + + +def test_should_get_RandomCellPhone_instance(anon): + cell_phone = anon.get_field("random_cell_phone", "id") + assert isinstance(cell_phone, RandomCellPhone) + assert cell_phone.render() == "`id` = LPAD(id, 13, 5)" + + +def test_should_get_RandomPhone_instance(anon): + phone = anon.get_field("random_phone", "id") + assert isinstance(phone, RandomPhone) + assert phone.render() == "`id` = LPAD(id, 12, 5)" + + +def test_should_get_RandomCpf_instance(anon): + cpf = anon.get_field("random_cpf", "id") + assert isinstance(cpf, RandomCpf) + assert cpf.render() == "`id` = LPAD(id, 11, 5)" From 92c4b55646907366b4b621af9c7f13e404382364 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 22:14:31 -0300 Subject: [PATCH 25/67] adding ipdb --- requirements_dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_dev.txt b/requirements_dev.txt index 5afa89d..7cdbfb2 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,3 +2,4 @@ pytest==3.5.1 pytest-mock==1.10.0 +ipdb==0.11 From 253f7ae5097281b014c8eb966ad3779e7991ece2 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 22:18:13 -0300 Subject: [PATCH 26/67] copy sample files to package --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 2841c40..2d284ab 100644 --- a/setup.py +++ b/setup.py @@ -38,5 +38,7 @@ description=__doc__, long_description=__doc__, py_modules=["anonymize"], + package_data={'': ['sample1.yml', 'sample2.yml']}, + include_package_data=True, **setup_params ) From 8164d2b4f58cb86c54eeabb1972be19332c9321e Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 22:19:34 -0300 Subject: [PATCH 27/67] refactoring code --- anonymize/__init__.py | 30 +++++++++++++++++++++++------- tests/test_anonymize.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 tests/test_anonymize.py diff --git a/anonymize/__init__.py b/anonymize/__init__.py index 433c6d5..b619d80 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# coding: utf-8 +from __future__ import print_function import os import yaml import logging @@ -16,31 +18,45 @@ class Anonymize(object): def __init__(self, file_name, sample): self._file_name = file_name self._sample = sample + self._run = True + + self.validate() def run(self): - with open(self.file_name) as handle: - a = AnonymizeSchemes(yaml.load(handle)) - a.build() + if self._run: + with open(self.file_name) as handle: + a = AnonymizeSchemes(yaml.load(handle)) + a.build() @property def file_name(self): - if self._sample: + if self.is_sample(): BASE_DIR = os.path.dirname(os.path.dirname( os.path.dirname(os.path.dirname(__file__)))) return os.path.join( BASE_DIR, 'anonymize', "sample{}.yml".format(self._sample)) return self._file_name + def is_sample(self): + return self._file_name is None and self._sample + + def validate(self): + if self._sample is None and self._file_name is None: + self._run = False + print('\033[91m' + "Invalid option.") + def main(): parser = OptionParser() - parser.add_option('-y', '--yaml') - parser.add_option('-s', '--sample', default=1) + parser.add_option('-y', '--yaml', help="YAML file to read data from.") + parser.add_option('--sample-one', const=1, action="store_const") + parser.add_option('--sample-two', const=2, action="store_const") + (options, __) = parser.parse_args() anonymize = Anonymize(**{ "file_name": options.yaml, - "sample": options.sample + "sample": options.sample_one or options.sample_two }) anonymize.run() diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py new file mode 100644 index 0000000..1dd0004 --- /dev/null +++ b/tests/test_anonymize.py @@ -0,0 +1,31 @@ +import os +import sys +import pytest +try: + from StringIO import StringIO +except: + from io import StringIO +from anonymize import Anonymize + + +@pytest.fixture +def sample(): + BASE_DIR = os.path.dirname(os.path.dirname(__file__)) + return os.path.join(BASE_DIR, 'anonymize', "sample1.yml") + + +@pytest.fixture +def stdout(): + old_stdout = sys.stdout + + result = StringIO() + sys.stdout = result + return old_stdout, result + + +def test_should_get_trucate_at_output(sample, stdout): + a = Anonymize(file_name=sample, sample="") + a.run() + + sys.stdout, result = stdout + assert "TRUNCATE `stats_collections_counts`;" in result.getvalue() From 6feccffd095c9231d63888bdf9111fd0fa78467a Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 13 May 2018 22:29:16 -0300 Subject: [PATCH 28/67] updating the documentation and add requires at setup --- README.md | 14 ++++++++++---- setup.py | 1 + 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b136875..c4fc851 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,13 @@ developing. This script can do a few things (see `anonymize.yml`): delete: private: Yes -### Usage - - python anonymize.py > anon.sql - cat anon.sql | mysql +#### Installation +```sh +pip install https://github.com/riquellopes/mysql-anonymous/tarball/master +``` +### CookBook +```sh + anonymize --sample-one + anonymize --sample-two + anonymize -y database.yml +``` diff --git a/setup.py b/setup.py index 2d284ab..1cf5663 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ platforms=['python >= 2.7'], description=__doc__, long_description=__doc__, + install_requires=["pyyaml"], py_modules=["anonymize"], package_data={'': ['sample1.yml', 'sample2.yml']}, include_package_data=True, From 0a10e1fdc287c89873c707e626ec4da255236456 Mon Sep 17 00:00:00 2001 From: Henrique Date: Mon, 14 May 2018 08:44:25 -0300 Subject: [PATCH 29/67] adding exception support for sql --- README.md | 14 +++++++++++++- anonymize/anonymize.py | 16 ++++++++++++++-- anonymize/sample1.yml | 4 +++- tests/test_anonymize.py | 1 + 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c4fc851..97ddf90 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,21 @@ developing. This script can do a few things (see `anonymize.yml`): database: tables: mytable: - delete: + nullify: private: Yes +* Apply rules exception in some cases: e.g. + ``UPDATE mytable SET cellphone=NULL WHERE id NOT IN(556, 889)``: + + database: + tables: + mytable: + exception: + - 556 + - 889 + nullify: + - cellphone + #### Installation ```sh pip install https://github.com/riquellopes/mysql-anonymous/tarball/master diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index f7aeff4..48f9df2 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -37,13 +37,25 @@ def create(self): for table, data in self._scheme.tables.iteritems(): updates = [] - anon = AnonymizeField(data, data.pop('primary_key', "id")) + primary_key, exception = data.pop('primary_key', "id"), data.pop('exception', []) + anon = AnonymizeField(data, primary_key) for n in anon.build(): updates.append(n.render()) if updates: - self.append('UPDATE `%s` SET %s' % (table, ', '.join(updates))) + self.append( + 'UPDATE `{}` SET {}{}'.format( + table, + ', '.join(updates), + self._sql_exception(primary_key, exception))) + + def _sql_exception(self, primary_key, exception): + where = "" + if exception: + where = " WHERE {primary_key} NOT IN({ids})".format( + primary_key=primary_key, ids=", ".join(map(str, exception))) + return where class AnonymizeScheme(object): diff --git a/anonymize/sample1.yml b/anonymize/sample1.yml index b969a06..36c8f4b 100644 --- a/anonymize/sample1.yml +++ b/anonymize/sample1.yml @@ -52,7 +52,9 @@ database: - versioncomments tables: addons: - primary_key: 1 + exception: + - 556 + - 889 nullify: [nominationmessage, paypal_id, charity_id] random_int: - average_daily_downloads diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 1dd0004..befbae0 100644 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -29,3 +29,4 @@ def test_should_get_trucate_at_output(sample, stdout): sys.stdout, result = stdout assert "TRUNCATE `stats_collections_counts`;" in result.getvalue() + assert " WHERE id NOT IN(556, 889)" in result.getvalue() From 0687936f9a92402a19046cbf705dbe05a5bbb483 Mon Sep 17 00:00:00 2001 From: Henrique Date: Mon, 14 May 2018 08:46:04 -0300 Subject: [PATCH 30/67] updating readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 97ddf90..365f267 100644 --- a/README.md +++ b/README.md @@ -18,15 +18,18 @@ developing. This script can do a few things (see `anonymize.yml`): * Delete rows based on simple rules: e.g. ``DELETE FROM mytable WHERE private = "Yes"``: + '''yml database: tables: mytable: nullify: private: Yes + ''' * Apply rules exception in some cases: e.g. ``UPDATE mytable SET cellphone=NULL WHERE id NOT IN(556, 889)``: + '''yml database: tables: mytable: @@ -35,6 +38,7 @@ developing. This script can do a few things (see `anonymize.yml`): - 889 nullify: - cellphone + ''' #### Installation ```sh From fd9e0764e08926ad62eaf720cecad518aa601615 Mon Sep 17 00:00:00 2001 From: Henrique Date: Mon, 14 May 2018 08:46:47 -0300 Subject: [PATCH 31/67] updating readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 365f267..9e7f338 100644 --- a/README.md +++ b/README.md @@ -18,18 +18,18 @@ developing. This script can do a few things (see `anonymize.yml`): * Delete rows based on simple rules: e.g. ``DELETE FROM mytable WHERE private = "Yes"``: - '''yml + ```yml database: tables: mytable: nullify: private: Yes - ''' + ``` * Apply rules exception in some cases: e.g. ``UPDATE mytable SET cellphone=NULL WHERE id NOT IN(556, 889)``: - '''yml + ```yml database: tables: mytable: @@ -38,7 +38,7 @@ developing. This script can do a few things (see `anonymize.yml`): - 889 nullify: - cellphone - ''' + ``` #### Installation ```sh From 55957828c244b3314d927ddb797ed1a386093a9f Mon Sep 17 00:00:00 2001 From: Henrique Date: Mon, 14 May 2018 08:47:31 -0300 Subject: [PATCH 32/67] updating readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e7f338..e95790b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build Status](https://travis-ci.org/riquellopes/mysql-anonymous.svg?branch=master)](https://travis-ci.org/riquellopes/mysql-anonymous) Contributors can benefit from having real data when they are -developing. This script can do a few things (see `anonymize.yml`): +developing. This script can do a few things (see `sample1.yml` or 'sample2.yml'): * Truncate any tables (logs, and other cruft which may have sensitive data) * Nullify fields (emails, passwords, etc) From 93bd9c0361bebe1e6936f6c4f7d75510db038d99 Mon Sep 17 00:00:00 2001 From: Henrique Date: Mon, 14 May 2018 08:49:04 -0300 Subject: [PATCH 33/67] updating readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e95790b..348eb3a 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,13 @@ developing. This script can do a few things (see `sample1.yml` or 'sample2.yml' - cellphone ``` -#### Installation +Installation +------------ ```sh pip install https://github.com/riquellopes/mysql-anonymous/tarball/master ``` -### CookBook +CookBook +-------- ```sh anonymize --sample-one anonymize --sample-two From 03162bac40bcbf8cbc9c42c568bcf8e48b73e49c Mon Sep 17 00:00:00 2001 From: Henrique Date: Tue, 15 May 2018 08:10:43 -0300 Subject: [PATCH 34/67] updating readme --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 348eb3a..6872a87 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build Status](https://travis-ci.org/riquellopes/mysql-anonymous.svg?branch=master)](https://travis-ci.org/riquellopes/mysql-anonymous) Contributors can benefit from having real data when they are -developing. This script can do a few things (see `sample1.yml` or 'sample2.yml'): +developing. This script can do a few things (see `sample1.yml` or `sample2.yml`): * Truncate any tables (logs, and other cruft which may have sensitive data) * Nullify fields (emails, passwords, etc) @@ -40,6 +40,17 @@ developing. This script can do a few things (see `sample1.yml` or 'sample2.yml' - cellphone ``` +* Define an other name for primary key of table: e.g. + ``UPDATE mytable SET `email` = CONCAT(user_id, '@example.com')``: + + ```yml + database: + tables: + primary_key: user_id + mytable: + random_email: email + ``` + Installation ------------ ```sh From a4246d657cb3749aa145c30320df78f9811b01d2 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Wed, 16 May 2018 19:18:29 -0300 Subject: [PATCH 35/67] fixing path of file --- anonymize/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/anonymize/__init__.py b/anonymize/__init__.py index b619d80..2f186dd 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -31,10 +31,10 @@ def run(self): @property def file_name(self): if self.is_sample(): - BASE_DIR = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(__file__)))) return os.path.join( - BASE_DIR, 'anonymize', "sample{}.yml".format(self._sample)) + os.path.dirname(os.path.dirname(__file__)), + 'anonymize', + "sample{}.yml".format(self._sample)) return self._file_name def is_sample(self): From ce0ab71263b15fa1e4ac2f545120d0e74122ead7 Mon Sep 17 00:00:00 2001 From: ronaldosantoscs Date: Wed, 20 Jun 2018 07:50:47 -0300 Subject: [PATCH 36/67] =?UTF-8?q?Inclus=C3=A3o=20de=20informa=C3=A7=C3=A3o?= =?UTF-8?q?=20sobre=20o=20CNPJ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6872a87..e6f86d5 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ developing. This script can do a few things (see `sample1.yml` or `sample2.yml` * Random Cell Phone * Random Phone * Random [CPF](https://pt.wikipedia.org/wiki/Cadastro_de_pessoas_f%C3%ADsicas) + * Random [CNPJ](https://pt.wikipedia.org/wiki/Cadastro_Nacional_da_Pessoa_Jur%C3%ADdica) * Email addresses * Usernames * Delete rows based on simple rules: e.g. From 1c63e5382d77e213ad9a6e45794cec4607e16927 Mon Sep 17 00:00:00 2001 From: ronaldosantoscs Date: Thu, 21 Jun 2018 09:23:25 -0300 Subject: [PATCH 37/67] Inclusion of CNPJ randomization --- anonymize/field.py | 4 ++++ tests/test_field_methods.py | 7 ++++++- tests/test_get_updates.py | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/anonymize/field.py b/anonymize/field.py index d23aef3..8a29fc0 100644 --- a/anonymize/field.py +++ b/anonymize/field.py @@ -44,6 +44,9 @@ class RandomPhone(Field): class RandomCpf(Field): sql_field = "`{field}` = LPAD({primary_key}, 11, 5)" +class RandomCnpj(Field): + sql_field = "`{field}` = LPAD({primary_key}, 14, 5)" + class HashValue(Field): sql_field = "`{field}` = MD5(CONCAT(@common_hash_secret, `{field}`))" @@ -68,6 +71,7 @@ def __init__(self, data, primary_key): "random_cell_phone": RandomCellPhone, "random_phone": RandomPhone, "random_cpf": RandomCpf, + "random_cnpj": RandomCnpj, "hash_value": HashValue, "hash_email": HashEmail, } diff --git a/tests/test_field_methods.py b/tests/test_field_methods.py index 1361901..2b9eeab 100644 --- a/tests/test_field_methods.py +++ b/tests/test_field_methods.py @@ -1,6 +1,6 @@ import pytest from anonymize.field import ( - AnonymizeField, Nullify, RandomCellPhone, RandomPhone, RandomCpf) + AnonymizeField, Nullify, RandomCellPhone, RandomPhone, RandomCpf, RandomCnpj) @pytest.fixture @@ -48,3 +48,8 @@ def test_should_get_RandomCpf_instance(anon): cpf = anon.get_field("random_cpf", "id") assert isinstance(cpf, RandomCpf) assert cpf.render() == "`id` = LPAD(id, 11, 5)" + +def test_should_get_RandomCnpj_instance(anon): + cnpj = anon.get_field("random_cnpj", "cnpj") + assert isinstance(cnpj, RandomCnpj) + assert cnpj.render() == "`cnpj` = LPAD(id, 14, 5)" \ No newline at end of file diff --git a/tests/test_get_updates.py b/tests/test_get_updates.py index 417221b..8ced7f5 100644 --- a/tests/test_get_updates.py +++ b/tests/test_get_updates.py @@ -14,3 +14,18 @@ def test_should_get_the_update_list(): r = ["UPDATE `user` SET `phone` = NULL, `ip` = INET_NTOA(RAND()*1000000000), `email` = CONCAT(id, '@example.com')"] assert data == r + + +def test_should_get_the_update_list_with_cnpj(): + data = AnonymizeUpdate(AnonymizeScheme("default", { + "tables": { + "user": { + "nullify": ["phone", ], + "random_cnpj": ["cnpj", ] + } + } + })) + + r = ['UPDATE `user` SET `phone` = NULL, `cnpj` = LPAD(id, 14, 5)'] + assert data == r + From 3443c635b060f7001f369570d187b59eabcbd792 Mon Sep 17 00:00:00 2001 From: Henrique Date: Thu, 28 Jun 2018 13:42:08 -0300 Subject: [PATCH 38/67] adding support to coverall --- .coveralls.yml | 1 + .travis.yml | 3 +++ Makefile | 2 +- requirements_dev.txt | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 0000000..2c40aaa --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +repo_token: N7bB5WM4m9W6LrOeG9TLy40LTqIYLpjUG diff --git a/.travis.yml b/.travis.yml index 4cde52b..c86c70f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,3 +9,6 @@ install: "make setup" script: - make test + +after_sucess: + - coveralls diff --git a/Makefile b/Makefile index ac20885..e92a5eb 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PYTEST=.venv/bin/pytest PYTHON=.venv/bin/python test:clean - PYTHONPATH=anonymize ${PYTEST} -s -v tests/${path} + PYTHONPATH=anonymize ${PYTEST} -s -v --cov=anonymize tests/${path} venv: virtualenv .venv diff --git a/requirements_dev.txt b/requirements_dev.txt index 7cdbfb2..46c8277 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,4 +2,7 @@ pytest==3.5.1 pytest-mock==1.10.0 +pytest-cov==2.5.1 ipdb==0.11 +coverage==4.0.3 +python-coveralls==2.9.1 From 6c9600b8ffa840f3d1e6a724a7af0f34f58ae3ed Mon Sep 17 00:00:00 2001 From: Henrique Date: Thu, 28 Jun 2018 13:46:05 -0300 Subject: [PATCH 39/67] adding the coveralls badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e6f86d5..5ec611a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![Build Status](https://travis-ci.org/riquellopes/mysql-anonymous.svg?branch=master)](https://travis-ci.org/riquellopes/mysql-anonymous) +[![Coverage Status](https://coveralls.io/repos/github/riquellopes/mysql-anonymous/badge.svg?branch=master)](https://coveralls.io/github/riquellopes/mysql-anonymous?branch=master) + Contributors can benefit from having real data when they are developing. This script can do a few things (see `sample1.yml` or `sample2.yml`): From d5c5fb7c989cf218f20b3575f10b3e760161b2f2 Mon Sep 17 00:00:00 2001 From: Henrique Date: Thu, 28 Jun 2018 13:46:26 -0300 Subject: [PATCH 40/67] adding the coveralls badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 5ec611a..c2ae790 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ ## Mysql Anonymous [![Build Status](https://travis-ci.org/riquellopes/mysql-anonymous.svg?branch=master)](https://travis-ci.org/riquellopes/mysql-anonymous) - [![Coverage Status](https://coveralls.io/repos/github/riquellopes/mysql-anonymous/badge.svg?branch=master)](https://coveralls.io/github/riquellopes/mysql-anonymous?branch=master) Contributors can benefit from having real data when they are From 6480ac4a1f8356e6dc01e286aa3cd05893c8b3bf Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 13:54:52 -0300 Subject: [PATCH 41/67] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c2ae790..fffe3e7 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build Status](https://travis-ci.org/riquellopes/mysql-anonymous.svg?branch=master)](https://travis-ci.org/riquellopes/mysql-anonymous) [![Coverage Status](https://coveralls.io/repos/github/riquellopes/mysql-anonymous/badge.svg?branch=master)](https://coveralls.io/github/riquellopes/mysql-anonymous?branch=master) +[![Requirements Status](https://requires.io/github/riquellopes/mysql-anonymous/requirements.svg?branch=master)](https://requires.io/github/riquellopes/mysql-anonymous/requirements/?branch=master) Contributors can benefit from having real data when they are developing. This script can do a few things (see `sample1.yml` or `sample2.yml`): From d1bd8c7a74c9edb499bf76991ddd2c616c590bec Mon Sep 17 00:00:00 2001 From: Henrique Date: Thu, 28 Jun 2018 14:00:40 -0300 Subject: [PATCH 42/67] changing version of pytest --- requirements_dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 46c8277..8f63620 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,6 +1,6 @@ -r requirements.txt -pytest==3.5.1 +pytest==3.6.2 pytest-mock==1.10.0 pytest-cov==2.5.1 ipdb==0.11 From 5cee78bd5178c28e750ff62c5387429cc4d267da Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 20:18:52 -0300 Subject: [PATCH 43/67] deleting coveralls file --- .coveralls.yml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 2c40aaa..0000000 --- a/.coveralls.yml +++ /dev/null @@ -1 +0,0 @@ -repo_token: N7bB5WM4m9W6LrOeG9TLy40LTqIYLpjUG From e68bf27da9d5f3c15373d98884585a133747247b Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 20:44:45 -0300 Subject: [PATCH 44/67] adding the extra commands for coverage --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e92a5eb..c985f09 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PYTEST=.venv/bin/pytest PYTHON=.venv/bin/python test:clean - PYTHONPATH=anonymize ${PYTEST} -s -v --cov=anonymize tests/${path} + PYTHONPATH=anonymize ${PYTEST} -s -v --cov=anonymize --cov-report term-missing tests/${path} venv: virtualenv .venv From f97c118715202991e31cf3497d128243864b0266 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 20:48:26 -0300 Subject: [PATCH 45/67] omit package tests --- .coveragerc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..9c734d2 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + */tests* \ No newline at end of file From c5e367126fdabcc99d00389db822f26c88874a8c Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 20:52:22 -0300 Subject: [PATCH 46/67] creating a new task --- .travis.yml | 2 +- Makefile | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c86c70f..895e3c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,4 +11,4 @@ script: - make test after_sucess: - - coveralls + - make coveralls diff --git a/Makefile b/Makefile index c985f09..22f6e61 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ PIP=.venv/bin/pip PYTEST=.venv/bin/pytest PYTHON=.venv/bin/python +COVERALLS=coveralls + test:clean PYTHONPATH=anonymize ${PYTEST} -s -v --cov=anonymize --cov-report term-missing tests/${path} @@ -17,3 +19,6 @@ clean: sample: PYTHONPATH=anonymize ${PYTHON} anonymize/__init__.py + +coveralls: + ${COVERALLS} From a4cdd3c4ac0e9bcd1034c3a766fdf8961a5f37ea Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 21:07:48 -0300 Subject: [PATCH 47/67] updating travis file --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 895e3c5..69db65c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,7 @@ python: install: "make setup" -script: - - make test +script: "make test" after_sucess: - make coveralls From b8c2d0875169b5cb70e9c93f0ae760040d18e039 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 21:12:47 -0300 Subject: [PATCH 48/67] updating travis file --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69db65c..534952a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: install: "make setup" -script: "make test" - -after_sucess: +script: + - make test - make coveralls From 33ea5e51a6b33acbe200e281fee0e67673fd4a57 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 28 Jun 2018 21:15:34 -0300 Subject: [PATCH 49/67] fix make --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 22f6e61..2e5b05c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PIP=.venv/bin/pip PYTEST=.venv/bin/pytest PYTHON=.venv/bin/python -COVERALLS=coveralls +COVERALLS=.venv/bin/coveralls test:clean From a13a5f977126f9277b7843f7c0df3e8f8c7ee1b6 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Wed, 10 Oct 2018 19:27:47 -0300 Subject: [PATCH 50/67] pep --- anonymize/field.py | 1 + 1 file changed, 1 insertion(+) diff --git a/anonymize/field.py b/anonymize/field.py index 8a29fc0..2caae54 100644 --- a/anonymize/field.py +++ b/anonymize/field.py @@ -44,6 +44,7 @@ class RandomPhone(Field): class RandomCpf(Field): sql_field = "`{field}` = LPAD({primary_key}, 11, 5)" + class RandomCnpj(Field): sql_field = "`{field}` = LPAD({primary_key}, 14, 5)" From a15868a806c35169a50777a3d2b00b528c85a69f Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Wed, 10 Oct 2018 19:37:54 -0300 Subject: [PATCH 51/67] pep --- tests/test_field_methods.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_field_methods.py b/tests/test_field_methods.py index 2b9eeab..b92f374 100644 --- a/tests/test_field_methods.py +++ b/tests/test_field_methods.py @@ -49,7 +49,8 @@ def test_should_get_RandomCpf_instance(anon): assert isinstance(cpf, RandomCpf) assert cpf.render() == "`id` = LPAD(id, 11, 5)" + def test_should_get_RandomCnpj_instance(anon): cnpj = anon.get_field("random_cnpj", "cnpj") assert isinstance(cnpj, RandomCnpj) - assert cnpj.render() == "`cnpj` = LPAD(id, 14, 5)" \ No newline at end of file + assert cnpj.render() == "`cnpj` = LPAD(id, 14, 5)" From 60a0a14409fd9bb61bdb4b9f7f1c05f6e98fd180 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Wed, 10 Oct 2018 19:45:59 -0300 Subject: [PATCH 52/67] adding support to Lorem Ipsum text field --- anonymize/field.py | 6 ++++++ tests/test_field_methods.py | 8 +++++++- tests/test_get_updates.py | 13 +++++++++++++ tests/test_lipsum.py | 1 + 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/test_lipsum.py diff --git a/anonymize/field.py b/anonymize/field.py index 2caae54..fe7d0c7 100644 --- a/anonymize/field.py +++ b/anonymize/field.py @@ -1,3 +1,4 @@ +# coding: utf-8 import logging logger = logging.getLogger('anonymize') @@ -57,6 +58,10 @@ class HashEmail(Field): sql_field = "`{field}` = CONCAT(MD5(CONCAT(@common_hash_secret, `{field}`)), '@example.com')" +class LoremIpsum(Field): + sql_field = """`{field}` = `Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum`""" + + class AnonymizeField(object): def __init__(self, data, primary_key): @@ -75,6 +80,7 @@ def __init__(self, data, primary_key): "random_cnpj": RandomCnpj, "hash_value": HashValue, "hash_email": HashEmail, + 'text_lorem_ipsum': LoremIpsum } def build(self): diff --git a/tests/test_field_methods.py b/tests/test_field_methods.py index b92f374..6d0d6bf 100644 --- a/tests/test_field_methods.py +++ b/tests/test_field_methods.py @@ -1,6 +1,6 @@ import pytest from anonymize.field import ( - AnonymizeField, Nullify, RandomCellPhone, RandomPhone, RandomCpf, RandomCnpj) + AnonymizeField, Nullify, RandomCellPhone, RandomPhone, RandomCpf, RandomCnpj, LoremIpsum) @pytest.fixture @@ -54,3 +54,9 @@ def test_should_get_RandomCnpj_instance(anon): cnpj = anon.get_field("random_cnpj", "cnpj") assert isinstance(cnpj, RandomCnpj) assert cnpj.render() == "`cnpj` = LPAD(id, 14, 5)" + + +def test_should_get_LoremIpsum_instance(anon): + lipsum = anon.get_field("text_lorem_ipsum", "text") + assert isinstance(lipsum, LoremIpsum) + assert "Lorem Ipsum" in lipsum.render() diff --git a/tests/test_get_updates.py b/tests/test_get_updates.py index 8ced7f5..06e9fd8 100644 --- a/tests/test_get_updates.py +++ b/tests/test_get_updates.py @@ -29,3 +29,16 @@ def test_should_get_the_update_list_with_cnpj(): r = ['UPDATE `user` SET `phone` = NULL, `cnpj` = LPAD(id, 14, 5)'] assert data == r + +def test_should_get_the_update_list_with_lipsum(): + data = AnonymizeUpdate(AnonymizeScheme("default", { + "tables": { + "user": { + "text_lorem_ipsum": ["text", ], + } + } + })) + + r = ["UPDATE `user` SET `text` = `Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum`"] + + assert data == r diff --git a/tests/test_lipsum.py b/tests/test_lipsum.py new file mode 100644 index 0000000..42cc8c6 --- /dev/null +++ b/tests/test_lipsum.py @@ -0,0 +1 @@ +from anonymize.anonymize import AnonymizeUpdate, AnonymizeScheme From daebf03300795cdc77545fb88afe8b53bb11e17f Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Wed, 10 Oct 2018 19:48:06 -0300 Subject: [PATCH 53/67] removing empty test --- tests/test_lipsum.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/test_lipsum.py diff --git a/tests/test_lipsum.py b/tests/test_lipsum.py deleted file mode 100644 index 42cc8c6..0000000 --- a/tests/test_lipsum.py +++ /dev/null @@ -1 +0,0 @@ -from anonymize.anonymize import AnonymizeUpdate, AnonymizeScheme From 2119a8ed3eaa164ad01f2eda6f92eae2eb695a9f Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Wed, 10 Oct 2018 19:50:00 -0300 Subject: [PATCH 54/67] updating the readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fffe3e7..2fb1ecd 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ developing. This script can do a few things (see `sample1.yml` or `sample2.yml` * Random [CNPJ](https://pt.wikipedia.org/wiki/Cadastro_Nacional_da_Pessoa_Jur%C3%ADdica) * Email addresses * Usernames + * Text [Loren Ipsum](https://www.lipsum.com/) + * Delete rows based on simple rules: e.g. ``DELETE FROM mytable WHERE private = "Yes"``: From 758d3d51a094c4f818bba2c86c695f81a2e8cd5c Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Wed, 10 Oct 2018 20:02:12 -0300 Subject: [PATCH 55/67] adding absolute_import package --- anonymize/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize/__init__.py b/anonymize/__init__.py index 2f186dd..6976d4d 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -1,11 +1,11 @@ #!/usr/bin/env python # coding: utf-8 -from __future__ import print_function +from __future__ import print_function, absolute_import import os import yaml import logging from optparse import OptionParser -from anonymize import AnonymizeSchemes +from .anonymize import AnonymizeSchemes logging.basicConfig( filename='anonymize.log', From 5055ae71c60c3a7eaccd56304eb78b29e98c2112 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 11 Oct 2018 08:41:54 -0300 Subject: [PATCH 56/67] upgrade the module to use items def --- anonymize/anonymize.py | 10 +++++++--- anonymize/field.py | 2 +- tests/test_get_updates.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/anonymize/anonymize.py b/anonymize/anonymize.py index 48f9df2..b7dfc07 100755 --- a/anonymize/anonymize.py +++ b/anonymize/anonymize.py @@ -2,6 +2,8 @@ import itertools import random from field import AnonymizeField +from collections import OrderedDict + common_hash_secret = "%016x" % (random.getrandbits(128)) @@ -23,10 +25,10 @@ def create(self): class AnonymizeDelete(AnonymizeBaseAction): def create(self): - for table, data in self._scheme.tables.iteritems(): + for table, data in self._scheme.tables.items(): if 'delete' in data: self.append('DELETE FROM `{}` WHERE '.format(table) + ' AND '.join( - ['`{}` = "{}"'.format(f, v) for f, v in data['delete'].iteritems()] + ['`{}` = "{}"'.format(f, v) for f, v in data['delete'].items()] )) @@ -35,9 +37,11 @@ class AnonymizeUpdate(AnonymizeBaseAction): def create(self): global common_hash_secret - for table, data in self._scheme.tables.iteritems(): + for table, data in self._scheme.tables.items(): updates = [] primary_key, exception = data.pop('primary_key', "id"), data.pop('exception', []) + data = OrderedDict(sorted(data.items(), key=lambda t: t[0])) + anon = AnonymizeField(data, primary_key) for n in anon.build(): diff --git a/anonymize/field.py b/anonymize/field.py index fe7d0c7..b03c33a 100644 --- a/anonymize/field.py +++ b/anonymize/field.py @@ -84,7 +84,7 @@ def __init__(self, data, primary_key): } def build(self): - for operation, details in self._data.iteritems(): + for operation, details in self._data.items(): if self._valid_operation(operation): for field in self._listify(details): yield self.get_field(operation, field) diff --git a/tests/test_get_updates.py b/tests/test_get_updates.py index 06e9fd8..f07f43e 100644 --- a/tests/test_get_updates.py +++ b/tests/test_get_updates.py @@ -7,12 +7,12 @@ def test_should_get_the_update_list(): "user": { "nullify": ["phone", ], "random_email": ["email", ], - "random_ip": ['ip'] + "random_ip": ['ip', ] } } })) - r = ["UPDATE `user` SET `phone` = NULL, `ip` = INET_NTOA(RAND()*1000000000), `email` = CONCAT(id, '@example.com')"] + r = ["UPDATE `user` SET `phone` = NULL, `email` = CONCAT(id, '@example.com'), `ip` = INET_NTOA(RAND()*1000000000)"] assert data == r From 69a8b2d9c2e467ccd404f6fde845f0db85ef9b8c Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 25 Oct 2018 23:22:49 -0300 Subject: [PATCH 57/67] creating new tests --- tests/conftest.py | 15 +++++++++++++ tests/test_anonymize.py | 13 ----------- tests/test_anonymize_scheme.py | 40 ++++++++++++++++++++++++++++++++++ tests/test_field_methods.py | 3 ++- 4 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_anonymize_scheme.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..ba7e1fb --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,15 @@ +import sys +import pytest +try: + from StringIO import StringIO +except: + from io import StringIO + + +@pytest.fixture +def stdout(): + old_stdout = sys.stdout + + result = StringIO() + sys.stdout = result + return old_stdout, result diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index befbae0..ad00704 100644 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -1,10 +1,6 @@ import os import sys import pytest -try: - from StringIO import StringIO -except: - from io import StringIO from anonymize import Anonymize @@ -14,15 +10,6 @@ def sample(): return os.path.join(BASE_DIR, 'anonymize', "sample1.yml") -@pytest.fixture -def stdout(): - old_stdout = sys.stdout - - result = StringIO() - sys.stdout = result - return old_stdout, result - - def test_should_get_trucate_at_output(sample, stdout): a = Anonymize(file_name=sample, sample="") a.run() diff --git a/tests/test_anonymize_scheme.py b/tests/test_anonymize_scheme.py new file mode 100644 index 0000000..f632691 --- /dev/null +++ b/tests/test_anonymize_scheme.py @@ -0,0 +1,40 @@ +import sys +from anonymize.anonymize import AnonymizeScheme, AnonymizeSchemes + + +def test_property_print_use_should_get_true(): + s = AnonymizeScheme("anything", { + "name": {} + }) + + assert s._print_use() + + +def test_property_print_use_should_get_false(): + s = AnonymizeScheme("anything", { + "names": {} + }) + + assert s._print_use() is False + + +def test_should_start_with_the_correct_header(stdout): + s = AnonymizeScheme("anything", { + "name": {} + }) + + s.create() + sys.stdout, result = stdout + + assert "USE `anything`\nSET FOREIGN_KEY_CHECKS=0;\nSET FOREIGN_KEY_CHECKS=1;\n\n" in result.getvalue() + + +def test_shoulg_get_true_and_empty_dict(): + s = AnonymizeSchemes({ + "databases": {} + }) + + databases = s._databases() + + assert s._print_use + assert databases == {} diff --git a/tests/test_field_methods.py b/tests/test_field_methods.py index 2b9eeab..b92f374 100644 --- a/tests/test_field_methods.py +++ b/tests/test_field_methods.py @@ -49,7 +49,8 @@ def test_should_get_RandomCpf_instance(anon): assert isinstance(cpf, RandomCpf) assert cpf.render() == "`id` = LPAD(id, 11, 5)" + def test_should_get_RandomCnpj_instance(anon): cnpj = anon.get_field("random_cnpj", "cnpj") assert isinstance(cnpj, RandomCnpj) - assert cnpj.render() == "`cnpj` = LPAD(id, 14, 5)" \ No newline at end of file + assert cnpj.render() == "`cnpj` = LPAD(id, 14, 5)" From 2972a9de9ff6b01e4ade0d7785ad52043605a885 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 28 Oct 2018 10:17:26 -0300 Subject: [PATCH 58/67] adding a silent task --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 2e5b05c..7fb2fa0 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +.SILENT: PIP=.venv/bin/pip PYTEST=.venv/bin/pytest PYTHON=.venv/bin/python From 6034d7eb6cfec261b1db9f4f3eda0ad86192a4d2 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Mon, 4 Mar 2019 21:14:08 -0300 Subject: [PATCH 59/67] fix the version of pyyaml --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c3726e8..59bc593 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pyyaml +PyYAML==3.12 From d736e088f85b51f409447767e3eddad7df5f6619 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Mon, 4 Mar 2019 22:15:14 -0300 Subject: [PATCH 60/67] adding filter --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 59bc593..34abb5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 +PyYAML==3.12 # rq.filter: !=3.13 From 2ba2b07708811e6508c744c68fb3f6f0a3de6cba Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Mon, 4 Mar 2019 22:20:59 -0300 Subject: [PATCH 61/67] removing filter --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 34abb5e..59bc593 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 # rq.filter: !=3.13 +PyYAML==3.12 From ea522b60b8a1fadfc7eb80586dd47d442ef7935d Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Mon, 4 Mar 2019 22:25:25 -0300 Subject: [PATCH 62/67] changing pyyaml version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 59bc593..4a28555 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -PyYAML==3.12 +PyYAML==3.13 From 6046fd1b42ca543d8805d20080391a87e5656069 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Sun, 10 Mar 2019 23:19:57 -0300 Subject: [PATCH 63/67] package secure --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4a28555..7d387e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -PyYAML==3.13 +PyYAML==4.2b2 diff --git a/setup.py b/setup.py index 1cf5663..fd65e38 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ platforms=['python >= 2.7'], description=__doc__, long_description=__doc__, - install_requires=["pyyaml"], + install_requires=["PyYAML==4.2b2"], py_modules=["anonymize"], package_data={'': ['sample1.yml', 'sample2.yml']}, include_package_data=True, From 6d5bcc0dbdc088e4397ecae3325513ce0d61c192 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 3 Feb 2022 14:24:55 -0300 Subject: [PATCH 64/67] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 2fb1ecd..9a18059 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![Build Status](https://travis-ci.org/riquellopes/mysql-anonymous.svg?branch=master)](https://travis-ci.org/riquellopes/mysql-anonymous) [![Coverage Status](https://coveralls.io/repos/github/riquellopes/mysql-anonymous/badge.svg?branch=master)](https://coveralls.io/github/riquellopes/mysql-anonymous?branch=master) -[![Requirements Status](https://requires.io/github/riquellopes/mysql-anonymous/requirements.svg?branch=master)](https://requires.io/github/riquellopes/mysql-anonymous/requirements/?branch=master) Contributors can benefit from having real data when they are developing. This script can do a few things (see `sample1.yml` or `sample2.yml`): From 7dad1042c8f80cb5925089a8e107237185804d6f Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 3 Feb 2022 21:26:42 -0300 Subject: [PATCH 65/67] incresed version --- Makefile | 2 +- requirements.txt | 2 +- requirements_dev.txt | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 7fb2fa0..cdd8bf4 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ test:clean PYTHONPATH=anonymize ${PYTEST} -s -v --cov=anonymize --cov-report term-missing tests/${path} venv: - virtualenv .venv + virtualenv .venv --python=python3 setup:venv ${PIP} install -U pip diff --git a/requirements.txt b/requirements.txt index 7d387e6..bee6c14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -PyYAML==4.2b2 +PyYAML==6.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 8f63620..e67a04a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,8 +1,8 @@ -r requirements.txt -pytest==3.6.2 -pytest-mock==1.10.0 -pytest-cov==2.5.1 -ipdb==0.11 -coverage==4.0.3 -python-coveralls==2.9.1 +pytest==6.2.5 +pytest-mock==3.7.0 +pytest-cov==3.0.0 +ipdb==0.13.9 +coverage==6.3.1 +python-coveralls==2.9.3 From 17d9113e3885fd3cdf3abd5ab8cefe5edd6ac8e9 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 3 Feb 2022 21:46:01 -0300 Subject: [PATCH 66/67] call class Loader --- anonymize/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize/__init__.py b/anonymize/__init__.py index 6976d4d..950ad27 100644 --- a/anonymize/__init__.py +++ b/anonymize/__init__.py @@ -2,8 +2,8 @@ # coding: utf-8 from __future__ import print_function, absolute_import import os -import yaml import logging +from yaml import Loader, load from optparse import OptionParser from .anonymize import AnonymizeSchemes @@ -25,7 +25,7 @@ def __init__(self, file_name, sample): def run(self): if self._run: with open(self.file_name) as handle: - a = AnonymizeSchemes(yaml.load(handle)) + a = AnonymizeSchemes(load(handle, Loader=Loader)) a.build() @property From 8b5e8bda1e87ae15bc14c797f5017a06527d6379 Mon Sep 17 00:00:00 2001 From: Henrique Lopes Date: Thu, 3 Feb 2022 21:48:59 -0300 Subject: [PATCH 67/67] added support to new python versions --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 534952a..8fc895f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,10 @@ language: python python: - - "2.7" - "3.5" - "3.6" + - "3.9" + - "3.10" install: "make setup"