Commit 0d6f6fdf authored by Joseph Walton-Rivers's avatar Joseph Walton-Rivers 🐦

rework system to support merging

parent 56db8d6a
Pipeline #1287 failed with stages
in 48 seconds
# Static jinja replacement
## Development install
./setup.py develop --user
import j2static.main
j2static.main.main()
##
# j2static generator
# Copyright (c) 2018, Joseph Walton-Rivers
##
import jinja2
import os
import tempfile
import subprocess
import shutil
import pathlib
class BaseGenerator(object):
""" """
def __init__(self, template_dir='.'):
self.jinja = self._mkenv(template_dir)
def render(self, path, context=dict()):
"""Build a single page and return the result"""
template = self.get_file(path)
return template.render(context)
def generate(self, path, outfile, context=dict()):
""" """
# ensure directory exists...
out_dir = outfile.parent
out_dir.mkdir(parents=True, exist_ok=True)
with open(outfile, 'w') as fp:
fp.write(self.render(path, context=context))
def filter(self, filename):
filepath = pathlib.Path(filename)
# if the filename starts with '_' we treat it as abstract
if filepath.name[0] == '_':
return False
return self.is_template(filepath)
def is_template(self, filepath):
return True
def get_files(self):
""" """
return self.jinja.list_templates(filter_func=self.filter)
def get_file(self, name):
return self.jinja.get_template(name)
class Website(BaseGenerator):
""" """
def is_template(self, filepath):
print(filepath.suffix)
return filepath.suffix in (".html", ".xhtml", ".xml")
def _mkenv(self, template_dir):
"""Make a jinja enviroment object suitable for a website"""
return jinja2.Environment(
loader=jinja2.FileSystemLoader(template_dir),
autoescape=True
)
class PlainText(BaseGenerator):
""" """
def _get_extentions(self):
return ("txt",)
def _mkenv(self, template_dir):
"""Make a jinja enviroment object suitable for plain text processing"""
return jinja2.Environment(
loader=jinja2.FileSystemLoader(template_dir),
autoescape=False
)
class Latex(BaseGenerator):
""" """
def _get_extentions(self):
return ("tex",)
def _mkenv(self, template_dir):
"""Make a jinja enviroment object suitable for plain text processing"""
return jinja2.Environment(
loader=jinja2.FileSystemLoader(template_dir),
autoescape=False
)
class TexPDF(BaseGenerator):
"""Use latexmk and a tempdir to make PDFs"""
def __init__(self):
self.texgen = Latex()
def _get_extentions(self):
return self.texgen._get_extentions()
def generate(self, path, outfile, context=None):
# ensure directory exists...
out_dir = outfile.parent
out_dir.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as td:
tmp_path = pathlib.Path(td)
self.texgen.generate(path, tmp_path / 'test.tex', context=context)
subprocess.run( ['latexmk', '-pdf', '-interaction=nonstopmode', 'test.tex'], cwd=td )
shutil.copy(os.path.join(td, 'test.pdf'), outfile)
def get_file(self, path):
""" """
return self.texgen.get_file(path)
_types = {
"html": Website,
"txt": PlainText,
"tex": Latex,
"pdf": TexPDF
}
def get_builder(build_type, template_dir):
return _types[build_type](template_dir)
......@@ -14,10 +14,19 @@ def main():
"""Main Entrypoint for CLI"""
parser = argparse.ArgumentParser("static site generator.")
parser.add_argument('action', choices=_options.keys())
# inputs
parser.add_argument('--template-dir', default="_templates/")
parser.add_argument('--data-dir', default="_data/")
parser.add_argument('--static-dir', default="_static/")
# outputs
parser.add_argument('--out-dir', default="site/")
args = parser.parse_args()
chosen_action = _options[args.action]
chosen_action()
chosen_action(args)
if __name__ == "__main__":
......
#! /usr/bin/env python3
##
# Extra CLI modes
##
import argparse
from j2static.tools import merge, marks
TEMPLATE_MAPPINGS = {
"pdf": "tex",
"html": "html",
"tex": "tex",
"txt": "txt"
}
def _parent():
parser = argparse.ArgumentParser(add_help=False)
# inputs
parser.add_argument('--template-dir', default="_templates/")
parser.add_argument('--data-dir', default="_data/")
parser.add_argument('--static-dir', default="_static/")
# outputs
parser.add_argument('--out-dir', default="site/")
return parser
def main():
"""J2 Mail Merge mode"""
parser = argparse.ArgumentParser(parents=[_parent()])
parser.add_argument("source", help='the source for the merge')
parser.add_argument("--mode", default="csv", choices=["csv", "dir"], help='the way in which source should be intepreted')
parser.add_argument("--builder", default="html", choices=TEMPLATE_MAPPINGS.keys(), help='the converter to use for outputting')
parser.add_argument("--template", default=None, help='the template to use')
parser.add_argument("--context", default=[], help='extra files to pass to the template')
args = parser.parse_args()
context = {}
if args.context:
context.update( merge.load_data(args.context) )
# if a template is not provided, but a builder is, try to guess the template name
if args.builder and not args.template:
args.template = "base."+TEMPLATE_MAPPINGS[args.builder]
if args.mode == "csv":
data = merge.merge_csv(args.source, callback=marks.deflatten)
elif args.mode == "dir":
data = merge.merge_dir(args.source)
else:
raise ValueError("unknown mode")
merge.write_dict(data, context=context, template=args.template, out_type=args.builder)
if __name__ == "__main__":
main()
#! /usr/bin/env python3
import json
import os
import pathlib
import shutil
from pathlib import Path
from j2static import build
from jinja2 import Environment, FileSystemLoader, select_autoescape
def generate(args, outdir='site/'):
out_path = pathlib.Path(outdir)
TEMPLATE_DIR = "templates"
DATA_DIR = "data"
SITE_DIR = "site"
ESCAPE = ('html', 'xml')
class TemplateEngine(object):
def __init__(self):
loader = FileSystemLoader(TEMPLATE_DIR)
self.env = Environment(
loader=loader,
autoescape=select_autoescape(ESCAPE)
)
def load_data(self, name):
"""Attempt to load a json encoded datafile"""
try:
template_name, ext = os.path.splitext(name)
path = os.path.join(DATA_DIR, template_name+".json")
with open(path) as f:
return json.load(f)
except FileNotFoundError:
return {}
def load_data_dir(self, data_dir):
"""Load all data files in a given directory"""
print("--data dir--")
for f in glob.iglob(data_dir + "*.json", recursive=True):
print(f)
def render(self, name):
template = self.env.get_template(name)
data = self.load_data(name)
return template.render(data)
def get_templates(self):
return self.env.list_templates(["html", "htm", "xml"])
def generate():
engine = TemplateEngine()
for template in engine.get_templates():
gen_path = os.path.join(SITE_DIR, template)
with open(gen_path, 'w') as f:
template_data = engine.render(template)
f.write(template_data)
generator = build.get_builder("html", args.template_dir)
for template in generator.get_files():
out_file = out_path / template
generator.generate(template, out_file)
#! /usr/bin/env python3
##
# Utils to emulate automark
##
def _walk(parts, data, parent):
if len(parts) == 0:
return data
if len(parts) == 1:
parent[parts[0]] = data
else:
head = parts[0]
if head not in parent:
parent[head] = dict()
_walk(parts[1:], data, parent[head])
def deflatten(row, sep="_"):
"""Deflatten filter converts a flat dictionary into a tree-like structure"""
out = dict()
for key, value in row.items():
parts = key.split(sep)
_walk(parts, value, out)
return out
def subtotal(cols, trigger="mark", key="total"):
"""Work out a total for answers formatted as: question => (trigger => score)"""
total = 0
for (question, val) in cols.items():
if not isinstance(val, dict):
continue
if trigger in val:
total += int(val[trigger])
if key and key not in cols:
cols[key] = total
return total
def total(cols, trigger="mark", key="total"):
"""Work out a total for answers formatted as: part => (question => (trigger => score))"""
total = 0
for (part, question) in cols.items():
total += subtotal(question)
if key and key not in cols:
cols[key] = total
return total
def test():
flattened = {
"p1_q1_mark": 3,
"p1_q1_comm": "cheese mark",
"p1_q2_mark": 0,
"p1_q2_comm": "fruit mark",
"p2_q1_mark": 3,
"p2_q1_comm": "cheese mark",
"p2_q2_mark": 0,
"p2_q2_comm": "fruit mark",
}
tree = deflatten(flattened)
subtotal = subtotal(tree)
print(tree)
##
# Mail Merge Mode for j2static
##
import os
import pathlib
import csv
import json
import pickle
from j2static.build import get_builder
def load_csv(fp):
"""Read a CSV file and put in into a list"""
reader = csv.DictReader(fp)
return [x for x in reader]
_decoders = {
'.json': json.load,
'.pkl': pickle.load,
'.csv': load_csv
}
def load_data(filename):
"""Load a file as something we can pass to jinja"""
filename = pathlib.Path(filename)
decoder = _decoders[filename.suffix]
with open(filename) as f:
return decoder(f)
def merge_csv(csv_file, callback=None, key=lambda x: x['id']):
"""Process a single file containing multiple records"""
data = dict()
with open(csv_file) as f:
reader = csv.DictReader(f)
for row in reader:
# give the user a chance to do some processing
if callback:
row = callback(row)
data[key(row)] = {'row': row }
return data
def merge_dir(pattern, root_path='.', callback=None, key=lambda x: x['id']):
"""Process multiple files in a directory, with one file containing one record"""
data_dict = dict()
path = pathlib.Path(root_path)
for item in path.glob(pattern):
suffix = item.suffix
if suffix not in _decoders:
print("sorry, don't know how to decode {} ...", suffix)
continue
row = _decoders[suffix](item)
if 'id' not in row:
row['id'] = item
# give the user a chance to do some processing
if callback:
row = callback(row)
data_dict[key(row)] = row
return data_dict
def write_dict(data_dict, template="base.html", context=dict(), out_type=None, out_dir='./out/'):
# try to get the builder...
if out_type == None:
name, out_type = os.path.splitext(template)
out_type = out_type[1:]
builder = get_builder(out_type, '.')
# iterate though the data
for (fname, data) in data_dict.items():
data.update(context)
path = pathlib.Path(out_dir) / "{}.{}".format(fname, out_type)
builder.generate(template, path, data)
......@@ -6,37 +6,53 @@
# in real time.
##
import j2static.main
import jinja2
from j2static.build import get_builder
from http.server import HTTPServer, BaseHTTPRequestHandler
import socketserver
PORT = 8082
_jinja = j2static.main.TemplateEngine()
PORT = 8080
TEMPLATE_DIR = "_templates/"
class TemplateHTTPServer(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server):
super().__init__(request, client_address, server)
print("constructor called")
def do_GET(self):
file_path = self.path
if file_path[-1] == "/":
file_path += "index.html"
builder = get_builder("html", TEMPLATE_DIR)
if builder.filter(file_path):
try:
data = builder.render(file_path)
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
file_path = self.path
if file_path == "/":
file_path = "index.html"
except jinja2.TemplateNotFound as e:
self.send_response(404)
self.send_header('Content-type', 'text/html')
self.end_headers()
data = "{e} does not exist as a template".format(e=e)
except jinja2.TemplateSyntaxError as e:
self.send_response(500)
self.send_header('Content-type', 'text/html')
self.end_headers()
_jinja.load_data_dir("data/")
data = "<h1>template error</h1> <p>{e.name}, line {e.lineno}: <samp>{e.message}</samp></p>".format(e=e)
else:
data = "not valid page"
data = _jinja.render(file_path)
self.wfile.write(data.encode())
def serve():
def serve(args):
Handler = TemplateHTTPServer
httpd = HTTPServer(('', PORT), Handler)
print("webpage will be served on http://localhost:8080")
httpd.serve_forever()
......@@ -28,11 +28,13 @@ setup(
'Development Status :: 3 - Alpha',
'Programming Language :: Python :: 3'
],
test_suite='tests',
packages=['j2static', ],
package_dir={'j2static': 'j2static'},
entry_points={
'console_scripts': [
'j2static=j2static.cli:main',
'j2merge=j2static.cli_merge:main'
],
},
)
# Fake test case to check that tox works
import unittest
import j2static.main
class TestDiscoveryDefaults(unittest.TestCase):
def setUp(self):
self.engine = j2static.main.TemplateEngine()
def test_abstract(self):
result = self.engine.is_abstract('_base.html')
return self.assertEqual(result, True)
def test_abstract_dir(self):
result = self.engine.is_abstract("/tmp/_base.html")
return self.assertEqual(result, True)
def test_abstract_not(self):
result = self.engine.is_abstract("base.html")
return self.assertEqual(result, False)
def test_abstract_not_dir(self):
result = self.engine.is_abstract("/tmp/base.html")
return self.assertEqual(result, False)
if __name__ == '__main__':
unittest.main()
......@@ -25,7 +25,7 @@ commands =
check-manifest --ignore tox.ini,tests*
python setup.py check -m -r -s
flake8 .
py.test tests
python -m unittest discover
[flake8]
exclude = .tox,*.egg,build,data
select = E,W,F
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment