diff --git a/development.ini-dist b/development.ini-dist index 815af3a..81e1c26 100644 --- a/development.ini-dist +++ b/development.ini-dist @@ -1,18 +1,9 @@ [app:main] use = egg:LBBulk -pyramid.reload_templates = true -pyramid.debug_authorization = false -pyramid.debug_notfound = false -pyramid.debug_routematch = false -pyramid.default_locale_name = en -pyramid.includes = - pyramid_tm - -sqlalchemy.url = postgresql://rest:rest@localhost/lbbulk - -domain = http://api.brlight.org -base_name = wmi +extract_dir = /tmp/extract-zip +json_filename = /zipmaluco/coleta.json +lightbase_url= http://0.0.0.0/api/basex/doc [server:main] use = egg:waitress#main diff --git a/lbbulk/__init__.py b/lbbulk/__init__.py index 46884a8..24bf7e8 100644 --- a/lbbulk/__init__.py +++ b/lbbulk/__init__.py @@ -1,21 +1,15 @@ -from pyramid.config import Configurator -from sqlalchemy import engine_from_config -from lbbulk.config.routing import make_routes -from pyramid_restler import includeme -from lbbulk.model import Base, metadata, DBSession +from pyramid.config import Configurator +from lbbulk import config def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ - config = Configurator(settings=settings) - config.scan('lbbulk') - engine = engine_from_config(settings, 'sqlalchemy.') - DBSession.configure(bind=engine) - Base.metadata.bind = engine - includeme(config) - config.include('pyramid_chameleon') - make_routes(config) - config.enable_POST_tunneling() - return config.make_wsgi_app() \ No newline at end of file + configurator = Configurator(settings=settings) + + config.setup_config(settings) + config.make_routes(configurator) + configurator.scan() + + return configurator.make_wsgi_app() diff --git a/lbbulk/config.py b/lbbulk/config.py new file mode 100644 index 0000000..dcbaa78 --- /dev/null +++ b/lbbulk/config.py @@ -0,0 +1,18 @@ + +def setup_config(settings): + + global EXTRACT_DIR + global JSON_FILENAME + global LIGHTBASE_URL + + EXTRACT_DIR = settings['extract_dir'] + JSON_FILENAME = settings['json_filename'] + LIGHTBASE_URL = settings['lightbase_url'] + +def make_routes(cfg): + + from lbbulk.view import zip_upload + + cfg.add_route('zip_upload', 'zip_upload', request_method='POST') + cfg.add_view(view=zip_upload, route_name='zip_upload') + diff --git a/lbbulk/config/__init__.py b/lbbulk/config/__init__.py deleted file mode 100644 index d310fdd..0000000 --- a/lbbulk/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# package \ No newline at end of file diff --git a/lbbulk/config/routing.py b/lbbulk/config/routing.py deleted file mode 100644 index 55b35d5..0000000 --- a/lbbulk/config/routing.py +++ /dev/null @@ -1,14 +0,0 @@ -# import lbbulk.model -from lbbulk.model.BulkUpload import BulkUploadContextFactory -from lbbulk.model.BulkSource import BulkSourceContextFactory -from lbbulk.view.restfulview import RegCustomView - -def make_routes(config): - """ - Create routes - """ - config.add_static_view('static', 'static', cache_max_age=3600) - config.add_route('home', '/') - config.add_restful_routes('source', BulkSourceContextFactory) - config.add_restful_routes('reg', BulkUploadContextFactory, - view=RegCustomView) diff --git a/lbbulk/model/BulkSource.py b/lbbulk/model/BulkSource.py deleted file mode 100644 index e40ebeb..0000000 --- a/lbbulk/model/BulkSource.py +++ /dev/null @@ -1,20 +0,0 @@ -from sqlalchemy import Table, Column, Integer, String -from pyramid_restler.model import SQLAlchemyORMContext -from lbbulk.model import Base, metadata, session - - -bulk_source = Table('lb_bulk_source', metadata, - Column('id_source', Integer, primary_key=True), - Column('name_source', String, nullable=False) - ) - - -# map to it -class BulkSource(Base): - __table__ = bulk_source - -class BulkSourceContextFactory(SQLAlchemyORMContext): - entity = BulkSource - - def session_factory(self): - return session diff --git a/lbbulk/model/BulkUpload.py b/lbbulk/model/BulkUpload.py deleted file mode 100644 index ce9221e..0000000 --- a/lbbulk/model/BulkUpload.py +++ /dev/null @@ -1,40 +0,0 @@ -from sqlalchemy import Table, Column, Integer, String, ForeignKey -from requests import get -from pyramid_restler.model import SQLAlchemyORMContext -from lbbulk.model import Base, metadata, session -import json - - -bulk_upload = Table('lb_bulk_upload', metadata, - Column('id_reg', Integer, primary_key=True), - Column('external_key', String, nullable=False), - Column('id_source', Integer, - ForeignKey('lb_bulk_source.id_source'), - nullable=False), - extend_existing=True - ) - -# map to it -class BulkUpload(Base): - __table__ = bulk_upload - - def verifica_registro(data): - # q = session.query(BulkUpload).filter_by(id_source=1, external_key=data['json_reg']['id_reg'] ) - get('localhost/') - registro_existe = q.first() - if sim: - registro_existe = True - else: - registro_existe = False - return registro_existe - - -class BulkUploadContextFactory(SQLAlchemyORMContext): - entity = BulkUpload - - def session_factory(self): - return session - - def get_member_id_as_string(self, member): - id = self.get_member_id(member) - return json.dumps(id, cls=self.json_encoder) \ No newline at end of file diff --git a/lbbulk/model/__init__.py b/lbbulk/model/__init__.py deleted file mode 100644 index 5c6348b..0000000 --- a/lbbulk/model/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from sqlalchemy import MetaData -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import scoped_session, sessionmaker - -metadata = MetaData() -Base = declarative_base() -DBSession = scoped_session(sessionmaker()) -session = DBSession \ No newline at end of file diff --git a/lbbulk/static/favicon.ico b/lbbulk/static/favicon.ico deleted file mode 100644 index 71f837c..0000000 Binary files a/lbbulk/static/favicon.ico and /dev/null differ diff --git a/lbbulk/static/footerbg.png b/lbbulk/static/footerbg.png deleted file mode 100644 index 1fbc873..0000000 Binary files a/lbbulk/static/footerbg.png and /dev/null differ diff --git a/lbbulk/static/headerbg.png b/lbbulk/static/headerbg.png deleted file mode 100644 index 0596f20..0000000 Binary files a/lbbulk/static/headerbg.png and /dev/null differ diff --git a/lbbulk/static/ie6.css b/lbbulk/static/ie6.css deleted file mode 100644 index b7c8493..0000000 --- a/lbbulk/static/ie6.css +++ /dev/null @@ -1,8 +0,0 @@ -* html img, -* html .png{position:relative;behavior:expression((this.runtimeStyle.behavior="none")&&(this.pngSet?this.pngSet=true:(this.nodeName == "IMG" && this.src.toLowerCase().indexOf('.png')>-1?(this.runtimeStyle.backgroundImage = "none", -this.runtimeStyle.filter = "progid:DXImageTransform.Microsoft.AlphaImageLoader(src='" + this.src + "',sizingMethod='image')", -this.src = "static/transparent.gif"):(this.origBg = this.origBg? this.origBg :this.currentStyle.backgroundImage.toString().replace('url("','').replace('")',''), -this.runtimeStyle.filter = "progid:DXImageTransform.Microsoft.AlphaImageLoader(src='" + this.origBg + "',sizingMethod='crop')", -this.runtimeStyle.backgroundImage = "none")),this.pngSet=true) -);} -#wrap{display:table;height:100%} diff --git a/lbbulk/static/middlebg.png b/lbbulk/static/middlebg.png deleted file mode 100644 index 2369cfb..0000000 Binary files a/lbbulk/static/middlebg.png and /dev/null differ diff --git a/lbbulk/static/pylons.css b/lbbulk/static/pylons.css deleted file mode 100644 index 4b1c017..0000000 --- a/lbbulk/static/pylons.css +++ /dev/null @@ -1,372 +0,0 @@ -html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockquote, pre, a, abbr, acronym, address, big, cite, code, del, dfn, em, font, img, ins, kbd, q, s, samp, small, strike, strong, sub, sup, tt, var, b, u, i, center, dl, dt, dd, ol, ul, li, fieldset, form, label, legend, table, caption, tbody, tfoot, thead, tr, th, td -{ - margin: 0; - padding: 0; - border: 0; - outline: 0; - font-size: 100%; /* 16px */ - vertical-align: baseline; - background: transparent; -} - -body -{ - line-height: 1; -} - -ol, ul -{ - list-style: none; -} - -blockquote, q -{ - quotes: none; -} - -blockquote:before, blockquote:after, q:before, q:after -{ - content: ''; - content: none; -} - -:focus -{ - outline: 0; -} - -ins -{ - text-decoration: none; -} - -del -{ - text-decoration: line-through; -} - -table -{ - border-collapse: collapse; - border-spacing: 0; -} - -sub -{ - vertical-align: sub; - font-size: smaller; - line-height: normal; -} - -sup -{ - vertical-align: super; - font-size: smaller; - line-height: normal; -} - -ul, menu, dir -{ - display: block; - list-style-type: disc; - margin: 1em 0; - padding-left: 40px; -} - -ol -{ - display: block; - list-style-type: decimal-leading-zero; - margin: 1em 0; - padding-left: 40px; -} - -li -{ - display: list-item; -} - -ul ul, ul ol, ul dir, ul menu, ul dl, ol ul, ol ol, ol dir, ol menu, ol dl, dir ul, dir ol, dir dir, dir menu, dir dl, menu ul, menu ol, menu dir, menu menu, menu dl, dl ul, dl ol, dl dir, dl menu, dl dl -{ - margin-top: 0; - margin-bottom: 0; -} - -ol ul, ul ul, menu ul, dir ul, ol menu, ul menu, menu menu, dir menu, ol dir, ul dir, menu dir, dir dir -{ - list-style-type: circle; -} - -ol ol ul, ol ul ul, ol menu ul, ol dir ul, ol ol menu, ol ul menu, ol menu menu, ol dir menu, ol ol dir, ol ul dir, ol menu dir, ol dir dir, ul ol ul, ul ul ul, ul menu ul, ul dir ul, ul ol menu, ul ul menu, ul menu menu, ul dir menu, ul ol dir, ul ul dir, ul menu dir, ul dir dir, menu ol ul, menu ul ul, menu menu ul, menu dir ul, menu ol menu, menu ul menu, menu menu menu, menu dir menu, menu ol dir, menu ul dir, menu menu dir, menu dir dir, dir ol ul, dir ul ul, dir menu ul, dir dir ul, dir ol menu, dir ul menu, dir menu menu, dir dir menu, dir ol dir, dir ul dir, dir menu dir, dir dir dir -{ - list-style-type: square; -} - -.hidden -{ - display: none; -} - -p -{ - line-height: 1.5em; -} - -h1 -{ - font-size: 1.75em; - line-height: 1.7em; - font-family: helvetica, verdana; -} - -h2 -{ - font-size: 1.5em; - line-height: 1.7em; - font-family: helvetica, verdana; -} - -h3 -{ - font-size: 1.25em; - line-height: 1.7em; - font-family: helvetica, verdana; -} - -h4 -{ - font-size: 1em; - line-height: 1.7em; - font-family: helvetica, verdana; -} - -html, body -{ - width: 100%; - height: 100%; -} - -body -{ - margin: 0; - padding: 0; - background-color: #fff; - position: relative; - font: 16px/24px NobileRegular, "Lucida Grande", Lucida, Verdana, sans-serif; -} - -a -{ - color: #1b61d6; - text-decoration: none; -} - -a:hover -{ - color: #e88f00; - text-decoration: underline; -} - -body h1, body h2, body h3, body h4, body h5, body h6 -{ - font-family: NeutonRegular, "Lucida Grande", Lucida, Verdana, sans-serif; - font-weight: 400; - color: #373839; - font-style: normal; -} - -#wrap -{ - min-height: 100%; -} - -#header, #footer -{ - width: 100%; - color: #fff; - height: 40px; - position: absolute; - text-align: center; - line-height: 40px; - overflow: hidden; - font-size: 12px; - vertical-align: middle; -} - -#header -{ - background: #000; - top: 0; - font-size: 14px; -} - -#footer -{ - bottom: 0; - background: #000 url(footerbg.png) repeat-x 0 top; - position: relative; - margin-top: -40px; - clear: both; -} - -.header, .footer -{ - width: 750px; - margin-right: auto; - margin-left: auto; -} - -.wrapper -{ - width: 100%; -} - -#top, #top-small, #bottom -{ - width: 100%; -} - -#top -{ - color: #000; - height: 230px; - background: #fff url(headerbg.png) repeat-x 0 top; - position: relative; -} - -#top-small -{ - color: #000; - height: 60px; - background: #fff url(headerbg.png) repeat-x 0 top; - position: relative; -} - -#bottom -{ - color: #222; - background-color: #fff; -} - -.top, .top-small, .middle, .bottom -{ - width: 750px; - margin-right: auto; - margin-left: auto; -} - -.top -{ - padding-top: 40px; -} - -.top-small -{ - padding-top: 10px; -} - -#middle -{ - width: 100%; - height: 100px; - background: url(middlebg.png) repeat-x; - border-top: 2px solid #fff; - border-bottom: 2px solid #b2b2b2; -} - -.app-welcome -{ - margin-top: 25px; -} - -.app-name -{ - color: #000; - font-weight: 700; -} - -.bottom -{ - padding-top: 50px; -} - -#left -{ - width: 350px; - float: left; - padding-right: 25px; -} - -#right -{ - width: 350px; - float: right; - padding-left: 25px; -} - -.align-left -{ - text-align: left; -} - -.align-right -{ - text-align: right; -} - -.align-center -{ - text-align: center; -} - -ul.links -{ - margin: 0; - padding: 0; -} - -ul.links li -{ - list-style-type: none; - font-size: 14px; -} - -form -{ - border-style: none; -} - -fieldset -{ - border-style: none; -} - -input -{ - color: #222; - border: 1px solid #ccc; - font-family: sans-serif; - font-size: 12px; - line-height: 16px; -} - -input[type=text], input[type=password] -{ - width: 205px; -} - -input[type=submit] -{ - background-color: #ddd; - font-weight: 700; -} - -/*Opera Fix*/ -body:before -{ - content: ""; - height: 100%; - float: left; - width: 0; - margin-top: -32767px; -} diff --git a/lbbulk/static/pyramid-small.png b/lbbulk/static/pyramid-small.png deleted file mode 100644 index a5bc0ad..0000000 Binary files a/lbbulk/static/pyramid-small.png and /dev/null differ diff --git a/lbbulk/static/pyramid.png b/lbbulk/static/pyramid.png deleted file mode 100644 index 347e055..0000000 Binary files a/lbbulk/static/pyramid.png and /dev/null differ diff --git a/lbbulk/static/transparent.gif b/lbbulk/static/transparent.gif deleted file mode 100644 index 0341802..0000000 Binary files a/lbbulk/static/transparent.gif and /dev/null differ diff --git a/lbbulk/templates/mytemplate.pt b/lbbulk/templates/mytemplate.pt deleted file mode 100644 index fada32d..0000000 --- a/lbbulk/templates/mytemplate.pt +++ /dev/null @@ -1,80 +0,0 @@ - - - - The Pyramid Web Framework - - - - - - - - - - -
-
-
-
pyramid
-
-
-
-
-

- Welcome to ${project}, an application generated by
- the Pyramid Web Framework. -

-
-
-
-
-
-

Restful Views

- -
- -
-
-
- - diff --git a/lbbulk/view.py b/lbbulk/view.py new file mode 100644 index 0000000..67886ad --- /dev/null +++ b/lbbulk/view.py @@ -0,0 +1,95 @@ + +import os +import cgi +import sys +import uuid +import json +import ijson +import shutil +import zipfile +import requests +from lbbulk import config +from multiprocessing import Process +from pyramid.view import view_config +from pyramid.response import Response + +@view_config(context=Exception) +def error_view(exc, request): + """ Customized Exception View + """ + #l = traceback.extract_tb(request.exc_info[2]) + exc_type, exc_obj, exc_tb = sys.exc_info() + exc_msg = exc_obj.args + if len(exc_obj.args) > 0: + exc_msg = exc_obj.args[0] + return Response(exc_msg, status=500) + +def zip_upload(request): + + file_ = request.params.get('file') + + if not isinstance(file_, cgi.FieldStorage): + return Response('File is not a zip file', status=500) + else: + ext_dir, json_file_path = extract_zip(file_) + process = Process(target=bulk_upload, args=(ext_dir, json_file_path, + config.LIGHTBASE_URL)) + process.start() + + return Response('OK') + +def extract_zip(zfile): + + identifier = str(uuid.uuid4()) + zpath = config.EXTRACT_DIR + '/' + identifier + '.zip' + zpath = os.path.abspath(zpath) + + try: + # write bytes to disk + with open(zpath, 'wb') as f: + f.write(zfile.file.read()) + except Exception as e: + raise Exception('Error while uploading file! %s' % e) + + ext_dir = os.path.abspath(config.EXTRACT_DIR + '/' + identifier) + + if not os.path.exists(ext_dir): + os.makedirs(ext_dir) + + try: + # extract zip file + with zipfile.ZipFile(zpath, "r") as z: + z.extractall(ext_dir) + except Exception as e: + raise Exception('Error while extracting zip file! %s' % e) + + # remove zip file + os.remove(zpath) + + json_file_path = ext_dir + '/' + config.JSON_FILENAME + json_file_path = os.path.abspath(json_file_path) + + if not os.path.exists(json_file_path): + raise Exception('Could not find any json file in zip file: %s' % + config.JSON_FILENAME) + + return ext_dir, json_file_path + +def bulk_upload(ext_dir, file_path, url): + + file_ = open(file_path, 'rb') + + try: + objects = ijson.items(file_, 'results.item') + computers = (computer for computer in objects) + except Exception as e: + print('Error While Reading JSON file: %s' % e) + + for computer in computers: + document = json.dumps(computer) + + response = requests.post(url, + data={'value': document}) + + shutil.rmtree(ext_dir) + diff --git a/lbbulk/view/__init__.py b/lbbulk/view/__init__.py deleted file mode 100644 index 8267c26..0000000 --- a/lbbulk/view/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from pyramid.response import Response -from pyramid.view import view_config - - -@view_config(route_name='home', renderer='../templates/mytemplate.pt') -def my_view(request): - return {'project': 'LBBulk'} \ No newline at end of file diff --git a/lbbulk/view/restfulview.py b/lbbulk/view/restfulview.py deleted file mode 100644 index 345e6e8..0000000 --- a/lbbulk/view/restfulview.py +++ /dev/null @@ -1,74 +0,0 @@ -from pyramid_restler.view import RESTfulView -from pyramid.response import Response -from lbbulk.model.BulkUpload import BulkUpload -import configparser -import json -import requests - -class RegCustomView(RESTfulView): - - def create_member(self): - member = self.context.create_member(self._get_data()) - id = self.context.get_member_id_as_string(member) - headers = {'Location': '/'.join((self.request.path, id))} - return Response(status=201, headers=headers) - - def _get_data(self): - """ Read json data from the post sent by some source """ - content_type = self.request.content_type - if content_type == 'application/json': - data = json.loads(self.request.body) - elif content_type == 'application/x-www-form-urlencoded': - data = dict(self.request.POST) - else: - data = self.request.params - data = self.send_to_lightbase(data) - # Must come with the external key as a value of - # 'data.json_reg.id_reg' - return data - - def send_to_lightbase(self, data): - """ Sends the registers to a neo-light base and returns a dict - with the external key and lighbase's id_reg """ - data['json_reg'] = json.loads(data['json_reg']) - # converte os filhos pra dicts -# existente = BulkUpload.verifica_registro(data) - registro = self.is_error(data) -# id_source = registro['id_source'] - id_source = 1 - external_key = registro['json_reg']['id_reg'] - registro['json_reg'].pop('id_reg', None) - registro['json_reg'].pop('name_source', None) - registro['json_reg'] = json.dumps(registro['json_reg']) - url = self.get_url_lightbase() -# if existente: -# r = requests.put(url, data=registro) -# id_reg = existente -# data = None - if False: - pass - else: - r = requests.post(url, data=registro) - id_reg = self.is_error_resp(r.json()) - data = { - 'id_source':id_source, - 'id_reg':id_reg, - 'external_key':external_key - } - return data - - def get_url_lightbase(self): #TODO - """ Returns url from config """ - domain = 'http://localhost/lightbase' - base_name = 'wmi' -# url = (settings['domain'] + '/' + settings['base_name'] + '/reg') - url = (domain + '/' + base_name + '/reg') - return url - - def is_error(self,data): #TODO - return data - - def is_error_resp(self,r): - if type(r) is dict: - raise TypeError('Lightbase error ' + str(r['_status']) + ': ' + r['_error_message']) - return r diff --git a/production.ini-dist b/production.ini-dist index 4bee257..81e1c26 100644 --- a/production.ini-dist +++ b/production.ini-dist @@ -1,18 +1,9 @@ [app:main] use = egg:LBBulk -pyramid.reload_templates = false -pyramid.debug_authorization = false -pyramid.debug_notfound = false -pyramid.debug_routematch = false -pyramid.default_locale_name = en -pyramid.includes = - pyramid_tm - -sqlalchemy.url = postgresql://rest:rest@localhost/lbbulk - -domain = http://api.brlight.org -base_name = wmi +extract_dir = /tmp/extract-zip +json_filename = /zipmaluco/coleta.json +lightbase_url= http://0.0.0.0/api/basex/doc [server:main] use = egg:waitress#main @@ -34,11 +25,11 @@ keys = console keys = generic [logger_root] -level = WARN +level = INFO handlers = console [logger_lbbulk] -level = WARN +level = DEBUG handlers = qualname = lbbulk diff --git a/setup.py b/setup.py index df3fa9f..4426770 100644 --- a/setup.py +++ b/setup.py @@ -9,18 +9,9 @@ with open(os.path.join(here, 'CHANGES.txt')) as f: CHANGES = f.read() requires = [ + 'ijson', 'pyramid', - 'SQLAlchemy', - 'transaction', - 'pyramid_tm', - 'pyramid_debugtoolbar', - 'zope.sqlalchemy', - 'waitress', - 'psycopg2', - 'pyramid_restler', - 'pyramid_chameleon', 'requests', - 'chameleon' ] setup(name='LBBulk', -- libgit2 0.21.2