From 43ae3103e1e9371b8e0ae4c63ca2b97b412280a2 Mon Sep 17 00:00:00 2001 From: Ross Mountjoy Date: Thu, 6 Feb 2020 10:57:45 -0500 Subject: [PATCH] started work on the platform/plugin system --- dashmachine/main/models.py | 25 +++++++++ dashmachine/main/read_config.py | 77 +++++++++++----------------- dashmachine/main/routes.py | 16 +++++- dashmachine/main/utils.py | 13 +++++ dashmachine/platform/rest.py | 47 +++++++++++++++-- dashmachine/static/js/main/home.js | 13 +++++ dashmachine/templates/main/home.html | 14 +++-- 7 files changed, 149 insertions(+), 56 deletions(-) diff --git a/dashmachine/main/models.py b/dashmachine/main/models.py index 7b61141..276a920 100644 --- a/dashmachine/main/models.py +++ b/dashmachine/main/models.py @@ -1,5 +1,11 @@ from dashmachine import db +rel_app_data_source = db.Table( + "rel_app_data_source", + db.Column("data_source_id", db.Integer, db.ForeignKey("data_sources.id")), + db.Column("app_id", db.Integer, db.ForeignKey("apps.id")), +) + class Files(db.Model): id = db.Column(db.Integer, primary_key=True) @@ -46,6 +52,25 @@ class ApiCalls(db.Model): value_template = db.Column(db.String()) +class DataSources(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String()) + platform = db.Column(db.String()) + args = db.relationship("DataSourcesArgs", backref="data_source") + apps = db.relationship( + "Apps", + secondary=rel_app_data_source, + backref=db.backref("data_sources", lazy="dynamic"), + ) + + +class DataSourcesArgs(db.Model): + id = db.Column(db.Integer, primary_key=True) + key = db.Column(db.String()) + value = db.Column(db.String()) + data_source_id = db.Column(db.Integer, db.ForeignKey("data_sources.id")) + + class Groups(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String()) diff --git a/dashmachine/main/read_config.py b/dashmachine/main/read_config.py index af5cbbc..26e652f 100644 --- a/dashmachine/main/read_config.py +++ b/dashmachine/main/read_config.py @@ -1,6 +1,6 @@ import os from configparser import ConfigParser -from dashmachine.main.models import Apps, ApiCalls, Groups +from dashmachine.main.models import Apps, ApiCalls, Groups, DataSources, DataSourcesArgs from dashmachine.settings_system.models import Settings from dashmachine.paths import user_data_folder from dashmachine import db @@ -21,6 +21,11 @@ def read_config(): except Exception as e: return {"msg": f"Invalid Config: {e}."} + ds_list = DataSources.query.all() + for ds in ds_list: + ds.apps.clear() + DataSources.query.delete() + DataSourcesArgs.query.delete() Apps.query.delete() ApiCalls.query.delete() Settings.query.delete() @@ -80,48 +85,21 @@ def read_config(): db.session.add(group) db.session.commit() - # API call creation + # Data source creation elif "platform" in config[section]: - api_call = ApiCalls() - api_call.name = section - if "resource" in config[section]: - api_call.resource = config[section]["resource"] - else: - return {"msg": f"Invalid Config: {section} does not contain resource."} - - if "method" in config[section]: - api_call.method = config[section]["method"] - else: - api_call.method = "GET" - - if "payload" in config[section]: - api_call.payload = config[section]["payload"] - else: - api_call.payload = None - - if "authentication" in config[section]: - api_call.authentication = config[section]["authentication"] - else: - api_call.authentication = None - - if "username" in config[section]: - api_call.username = config[section]["username"] - else: - api_call.username = None - - if "password" in config[section]: - api_call.password = config[section]["password"] - else: - api_call.password = None - - if "value_template" in config[section]: - api_call.value_template = config[section]["value_template"] - else: - api_call.value_template = section - - db.session.add(api_call) + data_source = DataSources() + data_source.name = section + data_source.platform = config[section]["platform"] + db.session.add(data_source) db.session.commit() - continue + for key, value in config[section].items(): + if key not in ["name", "platform"]: + arg = DataSourcesArgs() + arg.key = key + arg.value = value + arg.data_source = data_source + db.session.add(arg) + db.session.commit() else: # App creation @@ -157,11 +135,6 @@ def read_config(): else: app.open_in = "this_tab" - if "data_template" in config[section]: - app.data_template = config[section]["data_template"] - else: - app.data_template = None - if "groups" in config[section]: app.groups = config[section]["groups"] else: @@ -170,6 +143,18 @@ def read_config(): db.session.add(app) db.session.commit() + if "data_sources" in config[section]: + for config_ds in config[section]["data_sources"].split(","): + db_ds = DataSources.query.filter_by(name=config_ds.strip()).first() + if db_ds: + app.data_sources.append(db_ds) + db.session.merge(app) + db.session.commit() + else: + return { + "msg": f"Invalid Config: {section} has a data_source variable that doesn't exist." + } + group = Groups.query.filter_by(name="admin_only").first() if not group: group = Groups() diff --git a/dashmachine/main/routes.py b/dashmachine/main/routes.py index 9037548..546191a 100755 --- a/dashmachine/main/routes.py +++ b/dashmachine/main/routes.py @@ -4,8 +4,13 @@ from secrets import token_hex from htmlmin.main import minify from flask import render_template, url_for, redirect, request, Blueprint, jsonify from flask_login import current_user -from dashmachine.main.models import Files, Apps -from dashmachine.main.utils import get_rest_data, public_route, check_groups +from dashmachine.main.models import Files, Apps, DataSources +from dashmachine.main.utils import ( + get_rest_data, + public_route, + check_groups, + get_data_source, +) from dashmachine.settings_system.models import Settings from dashmachine.paths import cache_folder from dashmachine import app, db @@ -72,6 +77,13 @@ def load_rest_data(): return data_template +@main.route("/load_data_source", methods=["GET"]) +def load_data_source(): + data_source = DataSources.query.filter_by(id=request.args.get("id")).first() + data = get_data_source(data_source) + return data + + # ------------------------------------------------------------------------------ # TCDROP routes # ------------------------------------------------------------------------------ diff --git a/dashmachine/main/utils.py b/dashmachine/main/utils.py index 6cb2a45..71a7673 100755 --- a/dashmachine/main/utils.py +++ b/dashmachine/main/utils.py @@ -1,5 +1,6 @@ import os import subprocess +import importlib from shutil import copyfile from requests import get from configparser import ConfigParser @@ -143,3 +144,15 @@ def check_groups(groups, current_user): return True else: return False + + +def get_data_source(data_source): + data_source_args = [] + for arg in data_source.args: + data_source_args.append(row2dict(arg)) + data_source = row2dict(data_source) + module = importlib.import_module( + f"dashmachine.platform.{data_source['platform']}", "." + ) + platform = module.Platform(data_source, data_source_args) + return platform.process() diff --git a/dashmachine/platform/rest.py b/dashmachine/platform/rest.py index ebf0b98..c928fa7 100644 --- a/dashmachine/platform/rest.py +++ b/dashmachine/platform/rest.py @@ -1,6 +1,47 @@ +from requests import get + + class Platform: - def __init__(self, *args, **kwargs): - pass + def __init__(self, data_source, data_source_args): + self.data_source = data_source + self.name = data_source["name"] + + # parse the user's options from the config entries + for source_arg in data_source_args: + if source_arg.get("key") == "resource": + self.resource = source_arg.get("value") + + if source_arg.get("key") == "method": + self.method = source_arg.get("value") + else: + self.method = "GET" + + if source_arg.get("key") == "payload": + self.payload = source_arg.get("value") + + if source_arg.get("key") == "authentication": + self.authentication = source_arg.get("value") + + if source_arg.get("key") == "username": + self.username = source_arg.get("value") + + if source_arg.get("key") == "password": + self.password = source_arg.get("value") + + if source_arg.get("key") == "value_template": + self.value_template = source_arg.get("value") + else: + self.value_template = "value" + + if source_arg.get("key") == "data_template": + self.data_template = source_arg.get("value") + else: + self.value_template = self.name def process(self): - pass + if self.method.upper() == "GET": + try: + value = get(self.resource) + except: + pass + return self.name diff --git a/dashmachine/static/js/main/home.js b/dashmachine/static/js/main/home.js index c4bba42..47f6603 100644 --- a/dashmachine/static/js/main/home.js +++ b/dashmachine/static/js/main/home.js @@ -15,6 +15,19 @@ $( document ).ready(function() { }); }); + $(".data-source-container").each(function(e) { + var el = $(this); + $.ajax({ + url: el.attr('data-url'), + type: 'GET', + data: {id: el.attr('data-id')}, + success: function(data){ + el.closest('.col').find('.data-source-loading').addClass('hide'); + el.text(data); + } + }); + }); + $(".data-template").each(function(e) { var el = $(this); $.ajax({ diff --git a/dashmachine/templates/main/home.html b/dashmachine/templates/main/home.html index 766c392..d36cdd2 100755 --- a/dashmachine/templates/main/home.html +++ b/dashmachine/templates/main/home.html @@ -1,5 +1,5 @@ {% extends "main/layout.html" %} -{% from 'global_macros.html' import data %} +{% from 'global_macros.html' import data, preload_circle %} {% block page_vendor_css %} {% endblock page_vendor_css %} @@ -42,16 +42,20 @@
- {% if app.data_template %} + {% if app.data_sources.count() > 0 %}
-

- {{ app.data_template|safe }} -

+ {{ preload_circle() }} + {% for data_source in app.data_sources %} +

+

+ {% endfor %}
{% else %}