From d83e04af9dfbdff07a717107b11f4f2f32ae0ecd Mon Sep 17 00:00:00 2001 From: "a.chernenko" Date: Mon, 17 Nov 2025 20:41:49 +1000 Subject: [PATCH] =?UTF-8?q?=D0=A1=D0=BF=D0=B8=D1=81=D0=BE=D0=BA=20=D0=B2?= =?UTF-8?q?=D1=8B=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B8=20=D0=B2=D1=8B=D0=BD?= =?UTF-8?q?=D0=B5=D1=81=D0=B5=D0=BD=20=D0=B2=20=D0=BE=D1=82=D0=B4=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D0=BD=D1=8B=D0=B9=20=D1=84=D0=B0=D0=B9=D0=BB.=20?= =?UTF-8?q?=D0=B4=D0=BE=D0=B1=D0=B2=D0=BB=D0=B5=D0=BD=D0=B0=20=D0=B2=D0=BE?= =?UTF-8?q?=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD=D0=BE=D1=81=D1=82=D1=8C=20=D0=BE?= =?UTF-8?q?=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BE?= =?UTF-8?q?=D0=B4=D0=BD=D0=BE=D0=B3=D0=BE/=D0=BD=D0=B5=D1=81=D0=BA=D0=BE?= =?UTF-8?q?=D0=BB=D1=8C=D0=BA=D0=BE=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2.=20=D0=9F=D0=BE=D0=BD=D0=B8=D0=B6=D0=B5=D0=BD=20=D0=BF?= =?UTF-8?q?=D0=BE=D1=80=D0=BE=D0=B3=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=B2=D1=8B=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B8?= =?UTF-8?q?.=20=D0=A4=D0=B0=D0=B9=D0=BB=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=B2=D1=8B=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B8=20?= =?UTF-8?q?=D0=BC=D0=BE=D0=B6=D0=B5=D1=82=20=D1=81=D0=BE=D0=B4=D0=B5=D1=80?= =?UTF-8?q?=D0=B6=D0=B0=D1=82=D1=8C=20=D1=81=D1=81=D1=8B=D0=BB=D0=BA=D1=83?= =?UTF-8?q?=20=D0=BD=D0=B0=20=D1=84=D0=B0=D0=B9=D0=BB,=20=D0=B0=20=D0=BD?= =?UTF-8?q?=D0=B5=20json=20=D1=81=D1=82=D1=80=D1=83=D0=BA=D1=82=D1=83?= =?UTF-8?q?=D1=80=D1=83.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- download.py | 101 +++++++++++++++++++++++++++------------------------- list | 32 +++++++++++++++++ 2 files changed, 84 insertions(+), 49 deletions(-) create mode 100644 list diff --git a/download.py b/download.py index fab1702..b8fdb60 100755 --- a/download.py +++ b/download.py @@ -3,49 +3,10 @@ import re import os import sys +import ast import net_tree import requests -# массив выгружаемых списков -ip_list = { - 'RU': [ - # Большая часть RU сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=RU'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/ru.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/ru/aggregated.json'], 'ipv4': True, 'ipv6': False }, - # HLL LLC - { 'url': ['https://bgp.he.net/AS51115#_prefixes', 'https://ipinfo.io/widget/demo/AS51115?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS51115'], 'ipv4': True, 'ipv6': False }, - # STATIC - { 'static4': '188.130.255.0/24', 'ipv4': True, 'ipv6': False }, - ], - 'CHINA': [ - # Большая часть CH сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=CN'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/cn.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/cn/aggregated.json'], 'ipv4': True, 'ipv6': False }, - # Hong Kong - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=HK'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/hk.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/hk/aggregated.json'], 'ipv4': True, 'ipv6': False }, - # Alibaba (US) Technology Co., Ltd. - { 'url': ['https://bgp.he.net/AS45102#_prefixes', 'https://ipinfo.io/widget/demo/AS45102?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS45102'], 'ipv4': True, 'ipv6': False }, - ], - 'JAPAN': [ - # Большая часть KR сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=JP'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/jp.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/jp/aggregated.json'], 'ipv4': True, 'ipv6': False }, - ], - 'KOREA': [ - # Большая часть KR сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=KR'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/kr.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/kr/aggregated.json'], 'ipv4': True, 'ipv6': False }, - # LG DACOM Corporation - { 'url': ['https://bgp.he.net/AS3786#_prefixes', 'https://ipinfo.io/widget/demo/AS3786?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS3786'], 'ipv4': True, 'ipv6': False }, - ], -} - # компилируем регулярку поиска ipv4 адреса ipv4_find_str=re.compile(r"[^0-9.]?(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[1-9])\.(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[0-9])\.(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[0-9])\.(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[0-9])(/([0-9]{1}[0-9]*))?[^0-9.]?") @@ -103,13 +64,15 @@ def list_ip(c_dict: dict = []): if 'url' in list(c_list): # бежим весь список ссылок пока не код 200 for c_url in c_list['url']: - if (result:=requests.get(c_url)) and result.status_code == 200 and result.text: - print(f"URL: {c_url}") - # пополняем словарь ipv4_list - if ipv4: ipv4_list.update(ipv4_find(result.text,ipv4)) - # пополняем словарь ipv6_list - if ipv6: ipv6_list.update(ipv6_find(result.text,ipv6)) - break + try: + if (result:=requests.get(c_url)) and result.status_code == 200 and result.text: + print(f"URL: {c_url}") + # пополняем словарь ipv4_list + if ipv4: ipv4_list.update(ipv4_find(result.text,ipv4)) + # пополняем словарь ipv6_list + if ipv6: ipv6_list.update(ipv6_find(result.text,ipv6)) + break + except requests.exceptions.MissingSchema: pass print("Ошибка соединения") # если есть статичные записи ipv4 if ipv4 and 'static4' in list(c_list): @@ -155,6 +118,46 @@ def list_ip(c_dict: dict = []): # главная фукция if __name__ == "__main__": + # словарь выгружаемых списков + ip_list = dict() + try: + # если файл list содержет json структуру, парсим его + with open(list_file:=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'list'), "r") as file: + ip_list = ast.literal_eval(file.read()) + print(f"Список выгрузки из файла: {list_file}") + except (ValueError, SyntaxError): + try: + # если файл list ссылка, загружаем и парсим его + with open(list_file, "r") as file: + if (result:=requests.get(url_list_file:=file.readline().strip())) and result.status_code == 200 and result.text: + ip_list = ast.literal_eval(result.text) + print(f"Список выгрузки по url: {url_list_file}") + except requests.exceptions.MissingSchema: + print(f"Невалидный URL на список выгрузки", file=sys.stderr) + sys.exit(1) + except (ValueError, SyntaxError): + print(f"Ошибочная структура json", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Ошибка: {e}", file=sys.stderr) + sys.exit(1) + except FileNotFoundError: + print(f"Файл со списками не найден", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Ошибка: {e}", file=sys.stderr) + sys.exit(1) + + # если передан аргумент(ы) запуска, + # значит пытаемся обновить только указанные списки + if len(sys.argv) > 1: + ip_list = {k: v for k, v in ip_list.items() if k in sys.argv[1:]} + + # проверяем на пустой список выгрузки + if (not ip_list): + print(f"Список выгрузки пустой", file=sys.stderr) + sys.exit(1) + # создаем дерриктори. для сохранения outdir=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'unloading') if not os.path.exists(outdir): @@ -171,13 +174,13 @@ if __name__ == "__main__": print(f"Выгружаю список IP: {clist}") ipv4_list, ipv6_list=list_ip(ip_list[clist]) # сохраняем ipv4 - if ipv4_list and len(ipv4_list.splitlines()) >= ipv4_count_old * 0.7: + if ipv4_list and len(ipv4_list.splitlines()) >= ipv4_count_old * 0.5: # сохраняем в файл with open(ipv4_out_file, "w") as file: file.write(ipv4_list) print(f"Файл {ipv4_out_file} сохранён") # сохраняем ipv6 - if ipv6_list and len(ipv6_list.splitlines()) >= ipv6_count_old * 0.7: + if ipv6_list and len(ipv6_list.splitlines()) >= ipv6_count_old * 0.5: # сохраняем в файл with open(ipv6_out_file, "w") as file: file.write(ipv6_list) diff --git a/list b/list new file mode 100644 index 0000000..9bc6031 --- /dev/null +++ b/list @@ -0,0 +1,32 @@ +{ + 'RU': [ + # Большая часть RU сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=RU'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://ipv4.fetus.jp/ru.txt'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/ru/aggregated.json'], 'ipv4': True, 'ipv6': False }, + # HLL LLC + { 'url': ['https://bgp.he.net/AS51115#_prefixes', 'https://ipinfo.io/widget/demo/AS51115?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS51115'], 'ipv4': True, 'ipv6': False }, + # STATIC + { 'static4': '188.130.255.0/24', 'ipv4': True, 'ipv6': False }, + ], + 'CHINA': [ + # Большая часть CH сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=CN'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://ipv4.fetus.jp/cn.txt'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/cn/aggregated.json'], 'ipv4': True, 'ipv6': False }, + ], + 'JAPAN': [ + # Большая часть KR сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=JP'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://ipv4.fetus.jp/jp.txt'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/jp/aggregated.json'], 'ipv4': True, 'ipv6': False }, + ], + 'KOREA': [ + # Большая часть KR сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=KR'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://ipv4.fetus.jp/kr.txt'], 'ipv4': True, 'ipv6': False }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/kr/aggregated.json'], 'ipv4': True, 'ipv6': False }, + # LG DACOM Corporation + { 'url': ['https://bgp.he.net/AS3786#_prefixes', 'https://ipinfo.io/widget/demo/AS3786?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS3786'], 'ipv4': True, 'ipv6': False }, + ], +} \ No newline at end of file