From 87bba5279a799d4550775c06bf9d5aed526beec3 Mon Sep 17 00:00:00 2001 From: "a.chernenko" Date: Fri, 21 Nov 2025 17:44:10 +1000 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BF=D0=BE=D0=BB=D0=BD=D0=B0=D1=8F=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6=D0=BA=D0=B0=20=D0=B2=D1=8B?= =?UTF-8?q?=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B8=20ipv6=20=D0=BF=D0=BE=20?= =?UTF-8?q?=D0=B0=D0=BD=D0=B0=D0=BB=D0=BE=D0=B3=D0=B8=D0=B8=20=D1=81=20ipv?= =?UTF-8?q?4.=20=D0=9F=D0=BE=20=D1=83=D0=BC=D0=BE=D0=BB=D1=87=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D1=8E=20ipv6=20=D0=BD=D0=B5=20=D0=B2=D1=8B=D0=B3=D1=80?= =?UTF-8?q?=D1=83=D0=B6=D0=B0=D0=B5=D1=82=D1=81=D1=8F,=20=D1=81=D0=BC.=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=BD=D1=84=D0=B8=D0=B3=D1=83=D1=80=D0=B0=D1=86?= =?UTF-8?q?=D0=B8=D1=8E.=20=D0=98=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD?= =?UTF-8?q?=D0=B0=20=D1=81=D1=82=D1=80=D1=83=D0=BA=D1=82=D1=83=D1=80=D0=B0?= =?UTF-8?q?=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA=D0=B0=20=D0=B2=D1=8B=D0=B3?= =?UTF-8?q?=D1=80=D1=83=D0=B7=D0=BA=D0=B8.=20=D0=9F=D0=B0=D1=80=D0=B0?= =?UTF-8?q?=D0=BC=D0=B5=D1=82=D1=80=D1=8B=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8?= =?UTF-8?q?=D0=B3=D1=83=D1=80=D0=B0=D1=86=D0=B8=D0=B8=20=D0=B2=D1=8B=D0=BD?= =?UTF-8?q?=D0=B5=D1=81=D0=B5=D0=BD=D1=8B=20=D0=B3=D0=BB=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=BB=D1=8C=D0=BD=D0=BE=20=D0=B4=D0=BB=D1=8F=20=D0=BA=D0=B0?= =?UTF-8?q?=D0=B6=D0=B4=D0=BE=D0=B9=20=D0=B2=D1=8B=D0=B3=D1=80=D1=83=D0=B7?= =?UTF-8?q?=D0=BA=D0=B8,=20=D1=81=D0=BE=20=D0=B7=D0=BD=D0=B0=D1=87=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=D0=BC=D0=B8=20=D0=BF=D0=BE=20=D1=83=D0=BC?= =?UTF-8?q?=D0=BE=D0=BB=D1=87=D0=B0=D0=BD=D0=B8=D1=8E=20=D0=B8=20=D1=8F?= =?UTF-8?q?=D0=B2=D0=BB=D1=8F=D1=8E=D1=82=D1=81=D1=8F=20=D0=BD=D0=B5=20?= =?UTF-8?q?=D0=BE=D0=B1=D1=8F=D0=B7=D0=B0=D1=82=D0=B5=D0=BB=D1=8C=D0=BD?= =?UTF-8?q?=D1=8B=D0=BC=D0=B8.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- download.py | 125 ++++++++++++++++++---------- include/net_tree.py | 199 +++++++++++++++++++++++++++----------------- list | 96 +++++++++++---------- 3 files changed, 254 insertions(+), 166 deletions(-) diff --git a/download.py b/download.py index a923322..86dd087 100755 --- a/download.py +++ b/download.py @@ -5,12 +5,15 @@ import os import sys import ast import requests +import ipaddress from include import net_tree from collections import defaultdict from include.http_header import get_headers # компилируем регулярку поиска ipv4 адреса -ipv4_find_str=re.compile(r"[^0-9.]?(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[1-9])\.(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[0-9])\.(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[0-9])\.(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]|[0-9])(/([0-9]{1}[0-9]*))?[^0-9.]?") +ipv4_find_str=re.compile(r"(? size: continue + # проверка корректности IPv4 + try: + ipv4_obj=ipaddress.IPv4Address(ip_str) + except: + continue + listip[f"{ip_str}/{prefix}"] = [int(ipv4_obj), prefix] return listip -# метод сбора словаря ip адресов ipv4 из текста +# метод сбора словаря ip адресов ipv6 из текста def ipv6_find(strip:str, size:int): """ Метод сбора словаря ip адресов ipv4 из текста @@ -46,7 +47,19 @@ def ipv6_find(strip:str, size:int): ключ - имя сети значение - [адрес в int формате, размер сети] """ - return dict() + listip=dict() + for c in ipv6_find_str.finditer(strip): + ip_str = c.group(1) + prefix_str = c.group(2) + # определяем префикс + if (prefix:=int(prefix_str) if prefix_str else 128) > size: continue + # проверка корректности IPv4 + try: + ipv6_obj=ipaddress.IPv6Address(ip_str) + except: + continue + listip[f"{ip_str}/{prefix}"] = [int(ipv6_obj), prefix] + return listip # метод группировки словаря def get_dict_groups(input:dict): @@ -56,13 +69,16 @@ def get_dict_groups(input:dict): возвращает словарь сгрупированных данных данные встречающиеся один раз, не попадают в вывод """ + # если словарь короче 2х + if len(input) < 2: return {} + # строим битовые маски - name_to_bit = {name: 1 << i for i, name in enumerate(ipv4_dict.keys())} + name_to_bit = {name: 1 << i for i, name in enumerate(input.keys())} line_communities = defaultdict(set) line_mask = defaultdict(int) # походим по строкам и назначаем маску - for name, (communities, lines) in ipv4_dict.items(): + for name, (communities, lines) in input.items(): bit = name_to_bit[name] for line in lines: line_mask[line] |= bit @@ -84,10 +100,10 @@ def get_dict_groups(input:dict): bit_to_names[mask] = list_name # возвращаем словарь сгрупированных данных - return { bit_to_names[mask]: [ groups_communities[mask], set(groups[mask]) ] for mask in groups if "__" in bit_to_names[mask] } + return { bit_to_names[mask]: [ groups_communities[mask], set(groups[mask]) ] for mask in groups if "__" in bit_to_names[mask] and groups_communities[mask] and groups[mask] } # метод получения списка ip адресов -def list_ip(c_dict: dict = [], compress: bool = True): +def list_ip(c_list: list = []): """ Метод получения списка ip адресов возвращает кортеж из 2-х списков: ipv4 и ipv6 @@ -95,15 +111,22 @@ def list_ip(c_dict: dict = [], compress: bool = True): try: ipv4_list=dict() ipv6_list=dict() + # определяем, будем сжимать или нет + compress=c_list[0].get('compress', True) + # какие типы обрабытываем, от какого размера + # по умолчанию: + # ipv4 обрабатываем (<=24) + # ipv6 игнорируем (<=64) + # какие типы обрабытываем, от какого размера + ipv4 = False if not (ipv4:=c_list[0].get('ipv4', True)) else (ipv4 if type(ipv4) is int else 24) + ipv6 = False if not (ipv6:=c_list[0].get('ipv6', False)) else (ipv6 if type(ipv6) is int else 64) + # пробегаем словарь выгрузки - for c_list in c_dict: - # какие типы обрабытываем, от какого размера - ipv4 = False if 'ipv4' not in list(c_list) or not c_list['ipv4'] else (c_list['ipv4'] if type(c_list['ipv4']) is int else 24) - ipv6 = False if 'ipv6' not in list(c_list) or not c_list['ipv6'] else (c_list['ipv6'] if type(c_list['ipv6']) is int else 32) + for c_dict in c_list: # если есть источник ссылка - if 'url' in list(c_list): + if 'url' in list(c_dict): # бежим весь список ссылок пока не код 200 - for c_url in c_list['url']: + for c_url in c_dict['url']: try: session = requests.Session() session.headers.update(get_headers()) @@ -117,15 +140,15 @@ def list_ip(c_dict: dict = [], compress: bool = True): except requests.exceptions.MissingSchema: pass print("Ошибка соединения") # если есть статичные записи ipv4 - if ipv4 and 'static4' in list(c_list): + if ipv4 and 'static4' in list(c_dict): print(f"STATIC: IPv4") # пополняем словарь ipv4_list - ipv4_list.update(ipv4_find(str(c_list['static4']),ipv4)) + ipv4_list.update(ipv4_find(str(c_dict['static4']),ipv4)) # если есть статичные записи ipv6 - if ipv6 and 'static6' in list(c_list): + if ipv6 and 'static6' in list(c_dict): print(f"STATIC: IPv6") # пополняем словарь ipv6_list - ipv6_list.update(ipv6_find(str(c_list['static6']),ipv6)) + ipv6_list.update(ipv6_find(str(c_dict['static6']),ipv6)) # если ключ не сжимать if ipv4_list and not compress: @@ -133,7 +156,7 @@ def list_ip(c_dict: dict = [], compress: bool = True): # сжимаем подсети ipv4 elif ipv4_list and compress: # строим дерево - Root = net_tree.Node(net_tree.Net(0,0), 0) + Root = net_tree.Node(net_tree.Net(0, 0), 0) # пробегаем в цикле for c in ipv4_list.values(): # добавляем запись в дерево @@ -152,8 +175,20 @@ def list_ip(c_dict: dict = [], compress: bool = True): if ipv6_list and not compress: ipv6_list:str="".join([f"route {k} blackhole;\n" for k, v in ipv6_list.items() if isinstance(v, list)]) # сжимаем подсети ipv6 - if ipv6_list and compress: - None + elif ipv6_list and compress: + # строим дерево + Root = net_tree.Node(net_tree.Net(1 << 127, 0), 0) + # пробегаем в цикле + for c in ipv6_list.values(): + # добавляем запись в дерево + Root.addSubnet(net_tree.Node(net_tree.Net(c[0], c[1]), 1)) + Root.finishTreeFirst() + # жесткое сжатие в размер 30000 записей + #Root.collapseRoot(Root.real_ip_records_count - 30000) + # более мягкое сжатие + Root.collapse(1,Root.real_ip_records_count) + # возвращаем результат + ipv6_list:str=Root.returnCollapsedTree('route {addr}/{masklen} blackhole;') else: ipv6_list:bool=False @@ -216,12 +251,12 @@ if __name__ == "__main__": # удаляем старые файлы группировок [os.remove(path) for f in os.listdir(outdir) if "__" in f and os.path.isfile(path := os.path.join(outdir, f))] # обходим массив списков для выкрузки - for clist in ip_list: + for clist, value in ip_list.items(): # имена выходых файлов ipv4_out_file=f"{outdir}/{clist.lower()}_v4.txt" ipv6_out_file=f"{outdir}/{clist.lower()}_v6.txt" - # преобразуем запись комьюнити в список - ip_list[clist]['community']=list(ip_list[clist]['community'].split(",")) + # извлекаем community + community=[c for c in value[0].get('community', "").split(",") if c] # если передан аргумент(ы) запуска, # значит пытаемся обновить только указанные списки if len(sys.argv)==1 or clist in sys.argv[1:]: @@ -231,7 +266,7 @@ if __name__ == "__main__": ipv6_count_old = sum(1 for line in open(ipv6_out_file)) if os.path.isfile(ipv6_out_file) else 0 # выполняем выгрузку print(f"Выгружаю список IP: {clist}") - ipv4_list, ipv6_list=list_ip(ip_list[clist]['list'],ip_list[clist].get('compress', True)) + ipv4_list, ipv6_list=list_ip(value) # сохраняем ipv4 if ipv4_list and len(ipv4_list.splitlines()) >= ipv4_count_old * 0.5: # сохраняем в файл @@ -247,11 +282,11 @@ if __name__ == "__main__": # открываем файл выгрузки и пополняем словарь для группировки ipv4 if os.path.exists(ipv4_out_file): with open(ipv4_out_file, "r") as file: - ipv4_dict[clist] = [ip_list[clist]['community'],list(file.readlines())] + ipv4_dict[clist] = [community,list(file.readlines())] # открываем файл выгрузки и пополняем словарь для группировки ipv6 if os.path.exists(ipv6_out_file): with open(ipv6_out_file, "r") as file: - ipv6_dict[clist] = [ip_list[clist]['community'],list(file.readlines())] + ipv6_dict[clist] = [community,list(file.readlines())] # обновляем временный файл конфигурации ipv4 # из группировок @@ -266,7 +301,7 @@ if __name__ == "__main__": # список комьюнити маршрутов bgp_community=" ".join([f"bgp_community.add(({str(c).replace(':',',')}));" for c in sorted(v[0])]) with open(ipv4_bird2_m4, "a") as file: - file.write(f"protocol static static_{k.lower()}_v4 {{\n\tipv4 {{ import filter {{ {bgp_community} accept; }}; }};\n\tinclude \"{ipv4_out_file}\";\n}}\n") + file.write(f"protocol static static_{k.lower()}_v4 {{\n\tipv4 {{ import filter {{ {bgp_community}accept; }}; }};\n\tinclude \"{ipv4_out_file}\";\n}}\n") # обновляем временный файл конфигурации ipv6 # из группировок if ipv6_dict: @@ -280,21 +315,21 @@ if __name__ == "__main__": # список комьюнити маршрутов bgp_community=" ".join([f"bgp_community.add(({str(c).replace(':',',')}));" for c in sorted(v[0])]) with open(ipv6_bird2_m4, "a") as file: - file.write(f"protocol static static_{k.lower()}_v6 {{\n\tipv6 {{ import filter {{ {bgp_community} accept; }}; }};\n\tinclude \"{ipv6_out_file}\";\n}}\n") + file.write(f"protocol static static_{k.lower()}_v6 {{\n\tipv6 {{ import filter {{ {bgp_community}accept; }}; }};\n\tinclude \"{ipv6_out_file}\";\n}}\n") # дополняем временный файл конфигурации всей выгрузкой ipv4 и ipv6 - for clist in ip_list: + for clist, value in ip_list.items(): # имена выходых файлов ipv4_out_file=f"{outdir}/{clist.lower()}_v4.txt" ipv6_out_file=f"{outdir}/{clist.lower()}_v6.txt" - # список комьюнити маршрутов - bgp_community=" ".join([f"bgp_community.add(({str(c).replace(':',',')}));" for c in sorted(ip_list[clist]['community'])]) + # список комьюнити маршрутов + bgp_community=" ".join([f"bgp_community.add(({str(c).replace(':',',')}));" for c in sorted([c for c in value[0].get('community', "").split(",") if c])]) if os.path.exists(ipv4_out_file): with open(ipv4_bird2_m4, "a") as file: - file.write(f"protocol static static_{clist.lower()}_v4 {{\n\tipv4 {{ import filter {{ {bgp_community} accept; }}; }};\n\tinclude \"{ipv4_out_file}\";\n}}\n") + file.write(f"protocol static static_{clist.lower()}_v4 {{\n\tipv4 {{ import filter {{ {bgp_community}accept; }}; }};\n\tinclude \"{ipv4_out_file}\";\n}}\n") if os.path.exists(ipv6_out_file): with open(ipv6_bird2_m4, "a") as file: - file.write(f"protocol static static_{clist.lower()}_v6 {{\n\tipv4 {{ import filter {{ {bgp_community} accept; }}; }};\n\tinclude \"{ipv6_out_file}\";\n}}\n") + file.write(f"protocol static static_{clist.lower()}_v6 {{\n\tipv4 {{ import filter {{ {bgp_community}accept; }}; }};\n\tinclude \"{ipv6_out_file}\";\n}}\n") # проверяем, что временный файл конфигурации ipv4 не пустой, сохраняем в постоянный if os.path.exists(ipv4_bird2_m4) and os.path.getsize(ipv4_bird2_m4) != 0: diff --git a/include/net_tree.py b/include/net_tree.py index 8c546cd..44c097e 100644 --- a/include/net_tree.py +++ b/include/net_tree.py @@ -1,51 +1,127 @@ +# +# Unified IPv4/IPv6 Network Aggregator +# -BIG_MASK = (1 << 32) - 1 +def detect_ip_version(ip_int: int): + """Определяем IPv4 или IPv6 по величине числа""" + if ip_int <= 0xFFFFFFFF: + return 4, 32 + else: + return 6, 128 -def getMaskByMaskSize(mask_size): - return BIG_MASK ^ ((1 << (32 - mask_size)) - 1) -def getIpVolumeByMaskSize(mask_size): - return 1 << (32 - mask_size) +def get_mask_by_mask_size(mask_size, total_bits): + return ((1 << total_bits) - 1) ^ ((1 << (total_bits - mask_size)) - 1) + + +def get_ip_volume(mask_size, total_bits): + return 1 << (total_bits - mask_size) + + +def int_to_ipv4(n): + return ".".join(str((n >> (24 - 8*i)) & 0xFF) for i in range(4)) + + +def int_to_ipv6(n): + # разбор на 8 блоков по 16 бит + blocks = [(n >> (112 - 16*i)) & 0xFFFF for i in range(8)] + # убираем ведущие нули (схлопывание "::") + best_start = -1 + best_len = 0 + cur_start = -1 + cur_len = 0 + + for i in range(8): + if blocks[i] == 0: + if cur_start == -1: + cur_start = i + cur_len = 1 + else: + cur_len += 1 + else: + if cur_len > best_len: + best_len = cur_len + best_start = cur_start + cur_start = -1 + cur_len = 0 + + if cur_len > best_len: + best_len = cur_len + best_start = cur_start + + if best_len > 1: + new_blocks = [] + i = 0 + while i < 8: + if i == best_start: + new_blocks.append("") + i += best_len + else: + new_blocks.append(format(blocks[i], "x")) + i += 1 + res = ":".join(new_blocks) + # иногда получается ":::" → исправим + while ":::" in res: + res = res.replace(":::", "::") + return res + else: + return ":".join(format(b, "x") for b in blocks) + + +def int_to_ip(n, version): + return int_to_ipv4(n) if version == 4 else int_to_ipv6(n) + class Net: - __slots__ = ['mask_size', 'net', 'mask', 'ip_volume'] + __slots__ = ['mask_size', 'net', 'mask', 'ip_volume', 'version', 'total_bits'] def __init__(self, net: int, mask_size: int): + # Определяем IPv4/IPv6 + self.version, self.total_bits = detect_ip_version(net) self.mask_size = mask_size - self.net = net & getMaskByMaskSize(mask_size) - self.mask = getMaskByMaskSize(self.mask_size) - self.ip_volume = getIpVolumeByMaskSize(mask_size) - def hasSubnet(self, Net: 'Net'): - if Net.mask_size <= self.mask_size: return 0 - return self.net == Net.net & self.mask + self.mask = get_mask_by_mask_size(mask_size, self.total_bits) + self.net = net & self.mask + self.ip_volume = get_ip_volume(mask_size, self.total_bits) - def isSameNet(self, Net: 'Net'): - return (Net.mask_size == self.mask_size) and (Net.net == self.net) + def hasSubnet(self, other: 'Net'): + if other.version != self.version: + return 0 + if other.mask_size <= self.mask_size: + return 0 + return self.net == (other.net & self.mask) - def getCommonNet(self, OtherNet: 'Net', min_mask_size: int): + def isSameNet(self, other: 'Net'): + return ( + self.version == other.version and + self.mask_size == other.mask_size and + self.net == other.net + ) + + def getCommonNet(self, other: 'Net', min_mask_size: int): + if self.version != other.version: + return 0 if self.mask_size <= min_mask_size: return 0 - if OtherNet.mask_size <= min_mask_size: return 0 - for mask_size in range(min(self.mask_size, OtherNet.mask_size) - 1, min_mask_size - 1, -1): - mask = getMaskByMaskSize(mask_size) - if (self.net & mask) == (OtherNet.net & mask): - return Net(self.net, mask_size) + if other.mask_size <= min_mask_size: return 0 + + upper = min(self.mask_size, other.mask_size) - 1 + + for mask_size in range(upper, min_mask_size - 1, -1): + mask = get_mask_by_mask_size(mask_size, self.total_bits) + if (self.net & mask) == (other.net & mask): + return Net(self.net, mask_size) return 0 def getAsString(self, fmt='{addr}/{masklen}'): - net = self.net - mask = self.mask - addrbytes = [] - maskbytes = [] - for i in range(4): - addrbytes.append(str(net % 256)) - maskbytes.append(str(mask % 256)) - net = net >> 8 - mask = mask >> 8 - return fmt.format(addr='.'.join(reversed(addrbytes)), mask='.'.join(reversed(maskbytes)), masklen=self.mask_size) + return fmt.format( + addr=int_to_ip(self.net, self.version), + masklen=self.mask_size + ) + class Node: - __slots__ = ['net', 'child1', 'child2', 'is_real_net', 'real_ip_volume', 'real_ip_records_count', 'weight', 'max_child_weight', 'added_fake_ip_volume'] + __slots__ = ['net', 'child1', 'child2', 'is_real_net', 'real_ip_volume', + 'real_ip_records_count', 'weight', 'max_child_weight', 'added_fake_ip_volume'] def __init__(self, net: Net, is_real_net: int): self.net = net @@ -79,47 +155,23 @@ class Node: if Child and Child.addSubnet(NewNode): return 1 - if self.child1: - CommonNet = self.child1.net.getCommonNet(NewNode.net, self.net.mask_size + 1) - if CommonNet: - CommonNode = Node(CommonNet, 0) - CommonNode.addSubnet(NewNode) - CommonNode.addSubnet(self.child1) - self.child1 = CommonNode - return 1 - - if self.child2: - CommonNet = self.child2.net.getCommonNet(NewNode.net, self.net.mask_size + 1) - if CommonNet: - CommonNode = Node(CommonNet, 0) - CommonNode.addSubnet(NewNode) - CommonNode.addSubnet(self.child2) - self.child2 = CommonNode - return 1 + for child_attr in ('child1', 'child2'): + Child = getattr(self, child_attr) + if Child: + CommonNet = Child.net.getCommonNet(NewNode.net, self.net.mask_size + 1) + if CommonNet: + CommonNode = Node(CommonNet, 0) + CommonNode.addSubnet(NewNode) + CommonNode.addSubnet(Child) + setattr(self, child_attr, CommonNode) + return 1 if not self.child1: self.child1 = NewNode else: self.child2 = NewNode - return 1 - def printTree(self, level): - prefix = '' - for i in range(level): - prefix = prefix + ' ' - - if self.is_real_net: sign = '*' - elif self.weight == 0: sign = '.' - else: sign = '' - - print(prefix + self.net.getAsString() + ' ' + str(self.real_ip_records_count)) - - if self.child1: - self.child1.printTree(level + 1) - if self.child2: - self.child2.printTree(level + 1) - def finishTreeFirst(self): if self.is_real_net: self.real_ip_volume = self.net.ip_volume @@ -139,7 +191,6 @@ class Node: self.recalcWeight() def collapse(self, min_weight, max_net_delta): - # trying to collapse self if self.weight >= min_weight: self.weight = 0 self.max_child_weight = 0 @@ -150,6 +201,7 @@ class Node: net_delta = 0 fake_ip_delta = 0 self.max_child_weight = 0 + for Child in (self.child1, self.child2): if Child: if net_delta < max_net_delta and min_weight <= max(Child.weight, Child.max_child_weight): @@ -163,7 +215,6 @@ class Node: self.real_ip_records_count -= net_delta self.recalcWeight() - # trying to collapse self if self.weight >= min_weight: self.weight = 0 self.max_child_weight = 0 @@ -178,14 +229,6 @@ class Node: delta, fake_ip_volume = self.collapse(self.max_child_weight, required_net_delta) required_net_delta -= delta - def printCollapsedTree(self, fmt='{addr}/{masklen}'): - if self.is_real_net or self.weight == 0: - print(self.net.getAsString(fmt)) - else: - for Child in (self.child1, self.child2): - if Child: - Child.printCollapsedTree(fmt) - def returnCollapsedTree(self, fmt='{addr}/{masklen}'): if self.is_real_net or self.weight == 0: return self.net.getAsString(fmt) + "\n" @@ -198,8 +241,8 @@ class Node: def recalcWeight(self): fake_ip_delta = self.net.ip_volume - self.real_ip_volume - self.added_fake_ip_volume - if fake_ip_delta: - self.weight = (self.real_ip_records_count - 1) / fake_ip_delta + if fake_ip_delta > 0: + self.weight = (self.real_ip_records_count - 1) / (fake_ip_delta ** 0.5) else: self.weight = float('Inf') @@ -209,5 +252,5 @@ class Node: res = 0 for Child in (self.child1, self.child2): if Child: - res = res + Child.getNotRealIpCount() + res += Child.getNotRealIpCount() return res diff --git a/list b/list index 6e85302..f97f1db 100644 --- a/list +++ b/list @@ -1,44 +1,54 @@ { - 'RU': { - 'community': '65432:LOCATION,65432:200', - 'list': [ - # Большая часть RU сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=RU'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/ru.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/ru/aggregated.json'], 'ipv4': True, 'ipv6': False }, - # HLL LLC - { 'url': ['https://bgp.he.net/AS51115#_prefixes', 'https://ipinfo.io/widget/demo/AS51115?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS51115'], 'ipv4': True, 'ipv6': False }, - # STATIC - { 'static4': '188.130.255.0/24', 'ipv4': True, 'ipv6': False }, - ] - }, - 'CHINA': { - 'community': '65432:LOCATION,65432:201', - 'list': [ - # Большая часть CH сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=CN'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/cn.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/cn/aggregated.json'], 'ipv4': True, 'ipv6': False }, - ] - }, - 'JAPAN': { - 'community': '65432:LOCATION,65432:202', - 'list': [ - # Большая часть KR сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=JP'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/jp.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/jp/aggregated.json'], 'ipv4': True, 'ipv6': False }, - ] - }, - 'KOREA': { - 'community': '65432:LOCATION,65432:203', - 'list': [ - # Большая часть KR сегмента - { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=KR'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://ipv4.fetus.jp/kr.txt'], 'ipv4': True, 'ipv6': False }, - { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/kr/aggregated.json'], 'ipv4': True, 'ipv6': False }, - # LG DACOM Corporation - { 'url': ['https://bgp.he.net/AS3786#_prefixes', 'https://ipinfo.io/widget/demo/AS3786?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS3786'], 'ipv4': True, 'ipv6': False }, - ] - }, -} \ No newline at end of file + 'RU': [ + # параметры конфигурации должны быть в самом верху списка + { + 'ipv4': True, # не обязательный аргумент, по умолчанию True (24) + 'ipv6': True, # не обязательный аргумент, по умолчанию False (64) + 'compress': False, # не обязательный аргумент, по умолчанию True + 'community': '65432:LOCATION,65432:200' # не обязательный аргумент, по умолчанию пусто + }, + # Большая часть RU сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=RU'] }, + { 'url': ['https://ipv4.fetus.jp/ru.txt'] }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/ru/aggregated.json'] }, + # HLL LLC + { 'url': ['https://bgp.he.net/AS51115#_prefixes', 'https://ipinfo.io/widget/demo/AS51115?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS51115'] }, + # STATIC + { 'static4': '188.130.255.0/24' }, + ], + 'CHINA': [ + { + # пример без community + 'ipv4': True, + 'ipv6': False, + 'compress': True, + #'community': '65432:LOCATION,65432:201' + }, + # Большая часть CH сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=CN'] }, + { 'url': ['https://ipv4.fetus.jp/cn.txt'] }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/cn/aggregated.json'] }, + ], + 'JAPAN': [ + { + 'ipv6': True, + 'community': '65432:LOCATION,65432:202', + }, + # Большая часть KR сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=JP'] }, + { 'url': ['https://ipv4.fetus.jp/jp.txt'] }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/jp/aggregated.json'] }, + ], + 'KOREA': [ + { + 'ipv6': True, + 'community': '65432:LOCATION,65432:203' + }, + # Большая часть KR сегмента + { 'url': ['https://stat.ripe.net/data/country-resource-list/data.json?resource=KR'] }, + { 'url': ['https://ipv4.fetus.jp/kr.txt'] }, + { 'url': ['https://github.com/ipverse/rir-ip/blob/master/country/kr/aggregated.json'] }, + # LG DACOM Corporation + { 'url': ['https://bgp.he.net/AS3786#_prefixes', 'https://ipinfo.io/widget/demo/AS3786?dataset=asn', 'https://api.hackertarget.com/aslookup/?q=AS3786'] }, + ], +}