Commit eee293c0 authored by Max Lv's avatar Max Lv

Update GFW List

parent b634ef0d
#!/usr/bin/python
# -*- coding: utf-8 -*-
import pkgutil
import urlparse
import socket
import logging
from argparse import ArgumentParser
from datetime import date
__all__ = ['main']
def parse_args():
parser = ArgumentParser()
parser.add_argument('-i', '--input', dest='input', required=True,
help='path to gfwlist', metavar='GFWLIST')
parser.add_argument('-f', '--file', dest='output', required=True,
help='path to output acl', metavar='ACL')
return parser.parse_args()
def decode_gfwlist(content):
# decode base64 if have to
try:
return content.decode('base64')
except:
return content
def get_hostname(something):
try:
# quite enough for GFW
if not something.startswith('http:'):
something = 'http://' + something
r = urlparse.urlparse(something)
return r.hostname
except Exception as e:
logging.error(e)
return None
def add_domain_to_set(s, something):
hostname = get_hostname(something)
if hostname is not None:
if hostname.startswith('.'):
hostname = hostname.lstrip('.')
if hostname.endswith('/'):
hostname = hostname.rstrip('/')
if hostname:
s.add(hostname)
def parse_gfwlist(content):
gfwlist = content.splitlines(False)
domains = set()
for line in gfwlist:
if line.find('.*') >= 0:
continue
elif line.find('*') >= 0:
line = line.replace('*', '/')
if line.startswith('!'):
continue
elif line.startswith('['):
continue
elif line.startswith('@'):
# ignore white list
continue
elif line.startswith('||'):
add_domain_to_set(domains, line.lstrip('||'))
elif line.startswith('|'):
add_domain_to_set(domains, line.lstrip('|'))
elif line.startswith('.'):
add_domain_to_set(domains, line.lstrip('.'))
else:
add_domain_to_set(domains, line)
# TODO: reduce ['www.google.com', 'google.com'] to ['google.com']
return domains
def generate_acl(domains):
header ="""#
# GFW list from https://github.com/gfwlist/gfwlist/blob/master/gfwlist.txt
# updated on DATE
#
[bypass_all]
[proxy_list]
"""
header = header.replace('DATE', str(date.today()))
proxy_content = ""
ip_content = ""
for domain in sorted(domains):
try:
socket.inet_aton(domain)
ip_content += (domain + "\n")
except socket.error:
domain = domain.replace('.', '\.')
proxy_content += ('(^|\.)' + domain + '$\n')
proxy_content = header + ip_content + proxy_content
return proxy_content
def main():
args = parse_args()
with open(args.input, 'rb') as f:
content = f.read()
content = decode_gfwlist(content)
domains = parse_gfwlist(content)
acl_content = generate_acl(domains)
with open(args.output, 'wb') as f:
f.write(acl_content)
if __name__ == '__main__':
main()
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment