Commit ee2964d4 authored by Mart Lubbers's avatar Mart Lubbers

fixes issue #94

- Add iconv.sh to Dockerfile that converts all files to utf-8
- Changes cloogle_pygments.py to not try all encodings
parent 3e85429d
......@@ -27,3 +27,6 @@ RUN apt-get update && apt-get install -y python3.4 python3-pip mercurial
RUN hg clone https://bitbucket.org/camilstaps/pygments-main-3 /opt/pygments &&\
pip3 install -e /opt/pygments
COPY iconv.sh .
RUN bash iconv.sh && rm iconv.sh
#!/bin/bash
shopt -s globstar
for f in /opt/clean/lib/**/*.[id]cl; do
enc="$(file -bi "$f" | grep -Po '(?<=charset=).*')"
if [ "$enc" != 'us-ascii' -a "$enc" != 'binary' -a "$enc" != 'utf-8' ]; then
iconv -f "$enc" -t utf-8 < "$f" > "$f.tmp"
mv "$f.tmp" "$f"
echo "converted $f from $enc to utf-8"
fi
done
import urllib.parse
import os
import sys
import codecs
import pygments
import pygments.lexers
import pygments.formatters
......@@ -105,29 +104,26 @@ class CloogleHtmlFormatter(pygments.formatters.HtmlFormatter):
yield 1, ''.join(line)
for enc in ['latin1', 'utf8']:
try:
with open(sys.argv[1], 'rb') as f:
inp = u''
for l in f:
inp += l.decode(enc)
outp = pygments.highlight(
try:
with open(sys.argv[1], 'r', encoding='utf-8') as f:
inp = ''
for l in f:
inp += l
outp = pygments.highlight(
inp,
pygments.lexers.get_lexer_by_name('clean'),
CloogleHtmlFormatter(
full=False,
linenos=True,
linespans='line',
encoding='latin1',
encoding='utf-8',
hl_lines=[] if len(sys.argv) == 1 else [
int(a) for a in sys.argv[2:]],
))
print(codecs.decode(outp, 'utf-8', 'ignore'))
exit()
except Exception:
pass
print('<pre>{}</pre>'.format(traceback.format_exc()))
print(
'<p>Please open an issue <a href="https://github.com/clean-cloogle/clo'
'ogle/issues/new">here</a> with the exact way you got here and the tra'
'ce of this error.</p>')
sys.stdout.buffer.write(outp)
except:
print('<pre>{}</pre>'.format(traceback.format_exc()))
print(
'<p>Please open an issue <a href="https://github.com/clean-cloogle/clo'
'ogle/issues/new">here</a> with the exact way you got here and the tra'
'ce of this error.</p>')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment