Commit 4084a0bb authored by Camil Staps's avatar Camil Staps 🚀

Move encoding conversion to fetch_libs.sh so that the database is built on...

Move encoding conversion to fetch_libs.sh so that the database is built on UTF-8 files as well (prevents illegal characters in the JSON reults)
parent 980996bd
Pipeline #27920 passed with stages
in 18 minutes and 33 seconds
......@@ -5,7 +5,7 @@ set -ev
install_clean.sh 'base lib-platform lib-tcpip'
sed -i 's/cocl/cocl::-wmf/' /opt/clean/etc/IDEEnvs
PACKAGES="patch jq unzip z3"
PACKAGES="file patch unzip jq z3"
apt-get update -qq
apt-get install -qq $PACKAGES --no-install-recommends
......
......@@ -26,17 +26,6 @@ curl -Ls https://ftp.cs.ru.nl/Clean/builds/linux-x64/clean-base-linux-x64-latest
ln -s ../libs.json
../util/fetch_libs.sh /opt/clean/lib
# Convert files in weird encodings
shopt -s globstar
for f in /opt/clean/lib/**/*.[id]cl; do
enc="$(file -bi "$f" | grep -Po '(?<=charset=).*')"
if [ "$enc" != 'us-ascii' -a "$enc" != 'binary' -a "$enc" != 'utf-8' ]; then
iconv -f "$enc" -t utf-8 < "$f" > "$f.tmp"
mv "$f.tmp" "$f"
echo "converted $f from $enc to utf-8"
fi
done
# Build common problems index
apt-get install -qq python3 ca-certificates git
git clone https://gitlab.science.ru.nl/cloogle/common-problems /tmp/common-problems
......
......@@ -67,3 +67,13 @@ do
;;
esac
done
shopt -s globstar
for f in "$DEST"/**/*.[id]cl; do
enc="$(file -bi "$f" | grep -Po '(?<=charset=).*')"
if [ "$enc" != 'us-ascii' -a "$enc" != 'binary' -a "$enc" != 'utf-8' ]; then
iconv -f "$enc" -t utf-8 < "$f" > "$f.tmp"
mv "$f.tmp" "$f"
echo "converted $f from $enc to utf-8"
fi
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment