From 58dcfbf3ff7152016bf3906bbf5805e097a1ee0e Mon Sep 17 00:00:00 2001 From: Arthur de Jong Date: Sun, 22 Sep 2013 14:33:10 +0200 Subject: Provide function for template-based report rendering This uses the Jinja template engine to produce the report HTML files. This also renames the util module to output to better describe its purpose. --- webcheck/crawler.py | 2 +- webcheck/output.py | 135 +++++++++++++++++++++++++++++++++++++++++++ webcheck/plugins/__init__.py | 2 +- webcheck/util.py | 113 ------------------------------------ 4 files changed, 137 insertions(+), 115 deletions(-) create mode 100644 webcheck/output.py delete mode 100644 webcheck/util.py diff --git a/webcheck/crawler.py b/webcheck/crawler.py index 0c1a2da..abd1828 100644 --- a/webcheck/crawler.py +++ b/webcheck/crawler.py @@ -41,7 +41,7 @@ import urlparse from webcheck import config from webcheck.db import Session, Base, Link, truncate_db -from webcheck.util import install_file +from webcheck.output import install_file import webcheck.parsers from sqlalchemy import create_engine diff --git a/webcheck/output.py b/webcheck/output.py new file mode 100644 index 0000000..d49ed21 --- /dev/null +++ b/webcheck/output.py @@ -0,0 +1,135 @@ + +# output.py - utility functions for webcheck +# +# Copyright (C) 1998, 1999 Albert Hopkins (marduk) +# Copyright (C) 2002 Mike W. Meyer +# Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011, 2013 Arthur de Jong +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# The files produced as output from the software do not automatically fall +# under the copyright of the software, unless explicitly stated otherwise. + +"""Utility functions for generating the report.""" + +import codecs +import logging +import os +import shutil +import sys +import time +import urllib +import urlparse + +import jinja2 + +from webcheck import config +from webcheck.db import Link +import webcheck + + +logger = logging.getLogger(__name__) + + +def open_file(filename, is_text=True, makebackup=False): + """This returns an open file object which can be used for writing. This + file is created in the output directory. The output directory (stored in + config.OUTPUT_DIR is created if it does not yet exist. If the second + parameter is True (default) the file is opened as an UTF-8 text file.""" + # check if output directory exists and create it if needed + if not os.path.isdir(config.OUTPUT_DIR): + os.mkdir(config.OUTPUT_DIR) + # build the output file name + fname = os.path.join(config.OUTPUT_DIR, filename) + # check if file exists + if os.path.exists(fname): + if makebackup: + # create backup of original (overwriting previous backup) + os.rename(fname, fname + '~') + elif not config.OVERWRITE_FILES: + # ask to overwrite + try: + res = raw_input('webcheck: overwrite %s? [y]es, [a]ll, [q]uit: ' % fname) + except EOFError: + # bail out in case raw_input() failed + logger.exception('error reading response') + res = 'q' + res = res.lower() + ' ' + if res[0] == 'a': + config.OVERWRITE_FILES = True + elif res[0] != 'y': + raise SystemExit('Aborted.') + # open the file for writing + if is_text: + return codecs.open(fname, encoding='utf-8', mode='w') + else: + return open(fname, 'wb') + + +def install_file(source, is_text=False): + """Install the given file in the output directory. + If the is_text flag is set to true it is assumed the file is text, + translating line endings.""" + # figure out mode to open the file with + mode = 'r' + if is_text: + mode += 'U' + # check with what kind of argument we are called + scheme = urlparse.urlsplit(source)[0] + if scheme == 'file': + # this is a file:/// url, translate to normal path and open + source = urllib.url2pathname(urlparse.urlsplit(source)[2]) + elif scheme == '' and os.path.isabs(source): + # this is an absolute path, just open it as is + pass + elif scheme == '': + # this is a relavite path, try to fetch it from the python path + for directory in sys.path: + tst = os.path.join(directory, source) + if os.path.isfile(tst): + source = tst + break + # TODO: support more schemes here + # figure out the destination name + target = os.path.join(config.OUTPUT_DIR, os.path.basename(source)) + # test if source and target are the same + source = os.path.realpath(source) + if source == os.path.realpath(target): + logger.warn('attempt to overwrite %s with itself', source) + return + # open the input file + sfp = open(source, mode) + # create file in output directory (with overwrite question) + tfp = open_file(os.path.basename(source), is_text=is_text) + # copy contents + shutil.copyfileobj(sfp, tfp) + # close files + tfp.close() + sfp.close() + + +env = jinja2.Environment( + loader=jinja2.PackageLoader('webcheck'), + extensions=['jinja2.ext.autoescape'], + autoescape=True, + trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True) + + +def render(output_file, **kwargs): + """Render the output file with the specified context variables.""" + template = env.get_template(output_file) + fp = open_file(output_file) + fp.write(template.render(**kwargs)) + fp.close() diff --git a/webcheck/plugins/__init__.py b/webcheck/plugins/__init__.py index 4c534fd..4d032a4 100644 --- a/webcheck/plugins/__init__.py +++ b/webcheck/plugins/__init__.py @@ -51,7 +51,7 @@ import webcheck from webcheck import config from webcheck.db import Link from webcheck.parsers.html import htmlescape -from webcheck.util import open_file +from webcheck.output import open_file def _floatformat(f): diff --git a/webcheck/util.py b/webcheck/util.py deleted file mode 100644 index 911e3a2..0000000 --- a/webcheck/util.py +++ /dev/null @@ -1,113 +0,0 @@ - -# util.py - utility functions for webcheck -# -# Copyright (C) 1998, 1999 Albert Hopkins (marduk) -# Copyright (C) 2002 Mike W. Meyer -# Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011, 2013 Arthur de Jong -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -# -# The files produced as output from the software do not automatically fall -# under the copyright of the software, unless explicitly stated otherwise. - -import codecs -import logging -import os -import shutil -import sys -import urllib -import urlparse - -from webcheck import config - - -logger = logging.getLogger(__name__) - - -def open_file(filename, is_text=True, makebackup=False): - """This returns an open file object which can be used for writing. This - file is created in the output directory. The output directory (stored in - config.OUTPUT_DIR is created if it does not yet exist. If the second - parameter is True (default) the file is opened as an UTF-8 text file.""" - # check if output directory exists and create it if needed - if not os.path.isdir(config.OUTPUT_DIR): - os.mkdir(config.OUTPUT_DIR) - # build the output file name - fname = os.path.join(config.OUTPUT_DIR, filename) - # check if file exists - if os.path.exists(fname): - if makebackup: - # create backup of original (overwriting previous backup) - os.rename(fname, fname + '~') - elif not config.OVERWRITE_FILES: - # ask to overwrite - try: - res = raw_input('webcheck: overwrite %s? [y]es, [a]ll, [q]uit: ' % fname) - except EOFError: - # bail out in case raw_input() failed - logger.exception('error reading response') - res = 'q' - res = res.lower() + ' ' - if res[0] == 'a': - config.OVERWRITE_FILES = True - elif res[0] != 'y': - raise SystemExit('Aborted.') - # open the file for writing - if is_text: - return codecs.open(fname, encoding='utf-8', mode='w') - else: - return open(fname, 'wb') - - -def install_file(source, is_text=False): - """Install the given file in the output directory. - If the is_text flag is set to true it is assumed the file is text, - translating line endings.""" - # figure out mode to open the file with - mode = 'r' - if is_text: - mode += 'U' - # check with what kind of argument we are called - scheme = urlparse.urlsplit(source)[0] - if scheme == 'file': - # this is a file:/// url, translate to normal path and open - source = urllib.url2pathname(urlparse.urlsplit(source)[2]) - elif scheme == '' and os.path.isabs(source): - # this is an absolute path, just open it as is - pass - elif scheme == '': - # this is a relavite path, try to fetch it from the python path - for directory in sys.path: - tst = os.path.join(directory, source) - if os.path.isfile(tst): - source = tst - break - # TODO: support more schemes here - # figure out the destination name - target = os.path.join(config.OUTPUT_DIR, os.path.basename(source)) - # test if source and target are the same - source = os.path.realpath(source) - if source == os.path.realpath(target): - logger.warn('attempt to overwrite %s with itself', source) - return - # open the input file - sfp = open(source, mode) - # create file in output directory (with overwrite question) - tfp = open_file(os.path.basename(source), is_text=is_text) - # copy contents - shutil.copyfileobj(sfp, tfp) - # close files - tfp.close() - sfp.close() -- cgit v1.2.3