sbin/e2guardian-update-shallalists: Rewrite and rename to sbin/e2guardian-update-blacklists. Obtain blacklists from University of Toulouse now.

6 files changed, 160 insertions, 81 deletions

diff --git a/debian/itzks-systems-filter.install b/debian/itzks-systems-filter.install
index 5298025..f757290 100644
--- a/debian/itzks-systems-filter.install
+++ b/debian/itzks-systems-filter.install
@@ -1,5 +1,5 @@
 usr-lib-nagios-plugins/check_squid usr/lib/nagios/plugins/
 etc/cron.d/itzks-systems-filter
 etc/systemd/system/autofs.service
-sbin/e2guardian-update-shallalists usr/sbin/
+sbin/e2guardian-update-blacklists usr/sbin/
 sbin/itzks-puppet-refresh-ssl usr/sbin/
diff --git a/debian/itzks-systems-filter.manpages b/debian/itzks-systems-filter.manpages
index 5840249..af5580f 100644
--- a/debian/itzks-systems-filter.manpages
+++ b/debian/itzks-systems-filter.manpages
@@ -1 +1 @@
-man/e2guardian-update-shallalists.8
\ No newline at end of file
+man/e2guardian-update-blacklists.8
diff --git a/etc/cron.d/itzks-systems-filter b/etc/cron.d/itzks-systems-filter
index a0d8c31..84e862c 100644
--- a/etc/cron.d/itzks-systems-filter
+++ b/etc/cron.d/itzks-systems-filter
@@ -1,5 +1,5 @@
 MAILTO=root
 PATH=/usr/bin:/bin:/usr/sbin:/sbin
 
-#run e2guardian-update-shallalists script
-0 21   * * 0   [ -x /usr/sbin/e2guardian-update-shallalists ] && http_proxy="" https_proxy="" /usr/sbin/e2guardian-update-shallalists
+#run e2guardian-update-blacklists script
+0 21   * * 0   [ -x /usr/sbin/e2guardian-update-blacklists ] && http_proxy="" https_proxy="" /usr/sbin/e2guardian-update-blacklists
diff --git a/man/e2guardian-update-shallalists.8 b/man/e2guardian-update-blacklists.8
index 3bca2f8..61a30b8 100644
--- a/man/e2guardian-update-shallalists.8
+++ b/man/e2guardian-update-blacklists.8
@@ -5,18 +5,20 @@
 \\$2 \(la\\$1\(ra\\$3
 ..
 .if \n(.g .mso www.tmac
-.TH e2guardian-update-shalllists 8 "April 2019" "Version 2019.04.04.2" "IT-Zukunft Schule"
+.TH e2guardian-update-blacklists 8 "March 2023" "Version 2023.03.07.1" "IT-Zukunft Schule"
 .SH NAME
-e2guardian-update-shalllists \- Update content filter blacklists provided by Shalla Secure Services KG
+e2guardian-update-blacklists \- Update content filter blacklists provided by University of Toulouse
 .SH SYNOPSIS
 'nh
 .fi
 .ad l
-.B e2guardian-update-shalllists
+.B e2guardian-update-blacklists
 
 .SH DESCRIPTION
 This script is to be installed as a CRON job. It can be used to regularly
-install/update URL blacklists provided by Shalla Secure Services KG.
+install/update URL blacklists provided by University of Toulouse at
+ftp://ftp.ut-capitole.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz
+
 .PP
 The blacklists will be downloaded and installed to /var/lib/e2guardian/blacklists.
 .PP
@@ -24,8 +26,10 @@ To make these lists available to e2guardian, make sure to include those
 list via /etc/e2guardian/lists/bannedurllist and
 /etc/e2guardian/lists/bannedsitelist.
 .PP
+The backlists also contain sets of whitelists that can also be integrated in
+the e2guardian configuration.
 .SH OPTIONS
-\fBe2guardian-update-shalllists\fR has no known options.
+\fBe2guardian-update-blacklists\fR has no known options.
 .PP
 .SH AUTHOR
 This manual has been written by Mike Gabriel <mike.gabriel@das-netzwerkteam.de> for the IT-Zukunft Schule project
diff --git a/sbin/e2guardian-update-blacklists b/sbin/e2guardian-update-blacklists
new file mode 100755
index 0000000..c58c96b
--- /dev/null
+++ b/sbin/e2guardian-update-blacklists
@@ -0,0 +1,147 @@
+#!/bin/bash
+
+# Copyright (C) 2016 by Mike Gabriel <mike.gabriel@it-zukunft-schule.de>
+
+# This script is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This script is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the
+# Free Software Foundation, Inc.,
+# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+set -e
+
+unset http_proxy
+unset https_proxy
+unset ftp_proxy
+
+BLACKLIST_URL="http://dsi.ut-capitole.fr/blacklists/download/blacklists_for_dansguardian.tar.gz"
+
+function update_blacklists() {
+	mkdir -p /var/lib/e2guardian/
+	curl "${BLACKLIST_URL}" 1> /var/lib/e2guardian/blacklists.tar.gz 2>/dev/null
+
+	if [ -e /var/lib/e2guardian/.blacklists.new ]; then
+		rm -Rf /var/lib/e2guardian/.blacklists.new
+	fi
+
+	mkdir -p /var/lib/e2guardian/.blacklists.new
+
+	cd /var/lib/e2guardian/.blacklists.new
+	tar xzf /var/lib/e2guardian/blacklists.tar.gz
+
+	mv blacklists/* .
+	rmdir blacklists/
+
+	cd - 1>/dev/null
+
+	if [ -e /var/lib/e2guardian/blacklists.old ]; then
+		rm -Rf /var/lib/e2guardian/blacklists.old
+	fi
+
+	if [ -d /var/lib/e2guardian/blacklists ]; then
+		mv /var/lib/e2guardian/blacklists /var/lib/e2guardian/blacklists.old
+	fi
+
+	if [ ! -e /var/lib/e2guardian/blacklists ]; then
+		mv /var/lib/e2guardian/.blacklists.new /var/lib/e2guardian/blacklists
+
+		LANG=C date 1> /var/lib/e2guardian/last-updated.txt
+
+		chown root:root -Rf /var/lib/e2guardian/blacklists
+		chmod -Rf a+r /var/lib/e2guardian/blacklists
+		cd /var/lib/e2guardian/blacklists
+		find * -type d | while read dir; do
+			chmod a+x "${dir}"
+		done
+		cd - 1> /dev/null
+
+	fi
+
+	if [ -e /var/lib/e2guardian/blacklists.tar.gz ]; then
+		rm -f /var/lib/e2guardian/blacklists.tar.gz
+	fi
+}
+
+function update_whitelists() {
+	if [ -d /var/lib/e2guardian/whitelists.old ]; then
+		rm -Rf /var/lib/e2guardian/whitelists.old
+	fi
+	if [ -d /var/lib/e2guardian/whitelists ]; then
+		mv /var/lib/e2guardian/whitelists /var/lib/e2guardian/whitelists.old
+	fi
+
+	mkdir -p /var/lib/e2guardian/whitelists
+	find /var/lib/e2guardian/blacklists/*/usage | while read usage; do
+		# skip symlinked dirs
+		if [ -h "$(dirname "${usage}")" ]; then
+			continue
+		fi
+		if grep -q "white" < "${usage}" && ! grep -q "black" < "${usage}"; then
+			mv "$(dirname "${usage}")" /var/lib/e2guardian/whitelists
+		fi
+	done
+}
+
+function init_config() {
+	if [ ! -e "/etc/debian-edu/e2guardian_blacklisted_categories" ]; then
+		find  /var/lib/e2guardian/blacklists/* -maxdepth 1  -type d| cut -d "/" -f6 >> /etc/debian-edu/e2guardian_blacklisted_categories
+	fi
+	if [ ! -e "/etc/debian-edu/e2guardian_whitelisted_categories" ]; then
+		find  /var/lib/e2guardian/whitelists/* -maxdepth 1  -type d| cut -d "/" -f6 >> /etc/debian-edu/e2guardian_whitelisted_categories
+	fi
+}
+
+function rearrange_lists() {
+	cat "/etc/debian-edu/e2guardian_whitelisted_categories" | while read whitelisted; do
+		if [ -d "/var/lib/e2guardian/blacklists/${whitelisted}" ] && \
+		   [ ! -h "/var/lib/e2guardian/blacklists/${whitelisted}" ]; then
+			mv "/var/lib/e2guardian/blacklists/${whitelisted}" /var/lib/e2guardian/whitelists
+		fi
+	done
+
+	cat "/etc/debian-edu/e2guardian_blacklisted_categories" | while read blacklisted; do
+		if [ -d "/var/lib/e2guardian/whitelists/${blacklisted}" ] && \
+		   [ ! -h "/var/lib/e2guardian/whitelists/${blacklisted}" ]; then
+			mv "/var/lib/e2guardian/whitelists/${blacklisted}" /var/lib/e2guardian/blacklists
+		fi
+	done
+}
+
+function update_lists() {
+	action=${1}
+	type="${2}"
+	list="${3}"
+
+	sed -i "/etc/e2guardian/lists/${list}" -Ee "/(#|).Include<.*\/${action}s\/.*>$/d"
+
+	find /var/lib/e2guardian/${action}s/*/${type} -type f -maxdepth 1 2>/dev/null | sort | while read path; do
+		# ignore symlinks pointing to another category dir
+		if [ ! -h "$(echo $path | cut -d "/" -f 1-6)" ]; then
+			echo "#.Include<${path}>" >> "/etc/e2guardian/lists/${list}"
+		fi
+	done
+
+	cat /etc/debian-edu/e2guardian_${action}ed_categories | grep -v "^#" | while read category; do
+		sed -i "/etc/e2guardian/lists/${list}" -Ee "s/#(.Include<.*\/${category}\/${type})/\1/"
+	done
+}
+
+update_blacklists
+update_whitelists
+init_config
+rearrange_lists
+update_lists blacklist domains bannedsitelist
+update_lists blacklist urls bannedurllist
+update_lists blacklist expressions bannedregexpurllist
+update_lists whitelist domains exceptionsitelist
+update_lists whitelist urls exceptionurllist
+#invoke-rc.d e2guardian restart 1>/dev/null 2>/dev/null
diff --git a/sbin/e2guardian-update-shallalists b/sbin/e2guardian-update-shallalists
deleted file mode 100755
index 756e83c..0000000
--- a/sbin/e2guardian-update-shallalists
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-
-# Copyright (C) 2016 by Mike Gabriel <mike.gabriel@it-zukunft-schule.de>
-
-# This script is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This script is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the
-# Free Software Foundation, Inc.,
-# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
-
-set -e
-
-unset http_proxy
-unset https_proxy
-unset ftp_proxy
-
-SHALLA_BLACKLIST_URL="http://www.shallalist.de/Downloads/shallalist.tar.gz"
-
-mkdir -p /var/lib/e2guardian/
-curl "$SHALLA_BLACKLIST_URL" 1> /var/lib/e2guardian/blacklists.tar.gz 2>/dev/null
-
-if [ -e /var/lib/e2guardian/.blacklists.new ]; then
-        rm -Rf /var/lib/e2guardian/.blacklists.new
-fi
-
-mkdir -p /var/lib/e2guardian/.blacklists.new
-
-cd /var/lib/e2guardian/.blacklists.new
-tar xzf /var/lib/e2guardian/blacklists.tar.gz
-
-mv BL/* .
-rmdir BL/
-
-cd - 1>/dev/null
-
-if [ -e /var/lib/e2guardian/blacklists.old ]; then
-  rm -Rf /var/lib/e2guardian/blacklists.old
-fi
-
-if [ -d /var/lib/e2guardian/blacklists ]; then
-        mv /var/lib/e2guardian/blacklists /var/lib/e2guardian/blacklists.old
-fi
-
-if [ ! -e /var/lib/e2guardian/blacklists ]; then
-        mv /var/lib/e2guardian/.blacklists.new /var/lib/e2guardian/blacklists
-
-        LANG=C date 1> /var/lib/e2guardian/last-updated.txt
-
-        chown root:root -Rf /var/lib/e2guardian/blacklists
-        chmod -Rf a+r /var/lib/e2guardian/blacklists
-        cd /var/lib/e2guardian/blacklists
-        find * -type d | while read dir; do
-                chmod a+x "${dir}"
-        done
-        cd - 1> /dev/null
-
-        invoke-rc.d e2guardian restart 1>/dev/null 2>/dev/null
-fi
-
-if [ -e /var/lib/e2guardian/blacklists.tar.gz ]; then
-        rm -f /var/lib/e2guardian/blacklists.tar.gz
-fi
-