update

azure_count_ip_data.py

@@ -10,22 +10,26 @@ from typing import Dict, Set, Tuple, Optional
 from log_reader import RemoteLogReader
 
 # List of IP addresses we care about
-WHITELIST_IPS = [
-    "199.111.212.5",
-    "175.159.122.63",
-    "109.245.193.97",
-    "158.195.18.232",
-    "2607:fea8:4f40:4b00:e5b9:9806:6b69:233b",
-    "66.254.231.49",
-    "129.74.154.194", 
-    "175.196.44.217",
-    "2601:600:8d00:9510:1d77:b610:9358:f443",
-    "74.90.222.68",
-    "2a02:169:3e9:0:6ce8:e76f:faed:c830",
-    "70.50.179.57",
-    "2a02:842a:24:5a01:8cd6:5b22:1189:6035",
-    "2408:8418:6390:7603:40b:555f:774:a05d"
-]
+WHITELIST_IPS_DICT = {
+    "Chen Gong": ["199.111.212.5"],
+    "Juyong Jiang": ["175.159.122.63"],
+    "Kenneth Hamilton": ["109.245.193.97"],
+    "Marek Suppa": ["158.195.18.232"],
+    "Max Tian": ["2607:fea8:4f40:4b00:e5b9:9806:6b69:233b"],
+    "Mengzhao Jia": ["66.254.231.49"],
+    "Noah Ziems": ["129.74.154.194"],
+    "Sabina A": ["175.196.44.217"],
+    "Wenhao Yu": ["2601:600:8d00:9510:1d77:b610:9358:f443"],
+    "Vaisakhi Mishra": ["74.90.222.68"],
+    "Kumar Shridhar": ["129.132.145.250"],
+    "Viktor Gal": ["2a02:169:3e9:0:6ce8:e76f:faed:c830"],
+    "Guangyu Song": ["70.50.179.57"],
+    "Bhupesh Bishnoi": ["2a02:842a:24:5a01:8cd6:5b22:1189:6035"],
+    "Zheng Liu": ["2408:8418:6390:7603:40b:555f:774:a05d"]
+}
+
+# Flatten IP list for backward compatibility
+WHITELIST_IPS = [ip for ips in WHITELIST_IPS_DICT.values() for ip in ips]
 
 logging.basicConfig(level=logging.WARNING)
 log = logging.getLogger(__name__)
@@ -108,16 +112,19 @@ def get_file_data(content: str) -> Tuple[Optional[str], bool]:
         return None, False
 
 def count_files_per_ip(reader: 'RemoteLogReader', start_date_str: str = "2025_02_18") -> Dict[str, int]:
-    """Count files per IP address from the given start date"""
+    """Count files per name from the given start date"""
     # Convert start date string to datetime
     start_date = datetime.strptime(start_date_str, "%Y_%m_%d")
-    ip_counts = defaultdict(int)
+    name_counts = defaultdict(int)
     
     try:
         # Get current date for iteration
         current_date = start_date
         today = datetime.now()
         
+        # Create reverse mapping of IP to name
+        ip_to_name = {ip: name for name, ips in WHITELIST_IPS_DICT.items() for ip in ips}
+        
         while current_date <= today:
             date_str = current_date.strftime("%Y_%m_%d")
             
@@ -132,8 +139,9 @@ def count_files_per_ip(reader: 'RemoteLogReader', start_date_str: str = "2025_02
                         # Convert messages to file content format
                         content = '\n'.join(json.dumps(msg) for msg in messages)
                         ip, vote_conditions_met = get_file_data(content)
-                        if vote_conditions_met and ip:
-                            ip_counts[ip] += 1
+                        if vote_conditions_met and ip and ip in ip_to_name:
+                            name = ip_to_name[ip]
+                            name_counts[name] += 1
                             
             except Exception as e:
                 log.error(f"Error processing logs for date {date_str}: {e}")
@@ -144,10 +152,10 @@ def count_files_per_ip(reader: 'RemoteLogReader', start_date_str: str = "2025_02
     except Exception as e:
         log.error(f"Error accessing logs: {e}")
     
-    return dict(ip_counts)
+    return dict(name_counts)
 
-def download_files_by_ip(reader: 'RemoteLogReader', start_date_str: str = "2025_02_18", check_sandbox: bool = True) -> None:
-    """Download files and organize them by IP address
+def download_files_by_name(reader: 'RemoteLogReader', start_date_str: str = "2025_02_18", check_sandbox: bool = True) -> None:
+    """Download files and organize them by annotator name
     
     Args:
         reader: RemoteLogReader instance
@@ -158,6 +166,9 @@ def download_files_by_ip(reader: 'RemoteLogReader', start_date_str: str = "2025_
     data_dir = os.path.join(os.getcwd(), "data")
     os.makedirs(data_dir, exist_ok=True)
     
+    # Create reverse mapping of IP to name
+    ip_to_name = {ip: name for name, ips in WHITELIST_IPS_DICT.items() for ip in ips}
+    
     # Convert start date string to datetime
     start_date = datetime.strptime(start_date_str, "%Y_%m_%d")
     
@@ -186,11 +197,12 @@ def download_files_by_ip(reader: 'RemoteLogReader', start_date_str: str = "2025_
                     content = '\n'.join(json.dumps(msg) for msg in messages)
                     ip = get_ip_from_jsonl(content)
                     
-                    if ip:
-                        # Create directory structure for this IP
-                        ip_dir = os.path.join(data_dir, ip)
-                        valid_dir = os.path.join(ip_dir, "valid")
-                        invalid_dir = os.path.join(ip_dir, "invalid")
+                    if ip and ip in ip_to_name:
+                        name = ip_to_name[ip]
+                        # Create directory structure for this name
+                        name_dir = os.path.join(data_dir, name)
+                        valid_dir = os.path.join(name_dir, "valid")
+                        invalid_dir = os.path.join(name_dir, "invalid")
                         os.makedirs(valid_dir, exist_ok=True)
                         os.makedirs(invalid_dir, exist_ok=True)
                         
@@ -226,21 +238,21 @@ def main():
     reader = RemoteLogReader()
     
     # Add argument parser for optional parameters
-    parser = argparse.ArgumentParser(description='Download and organize conversation files by IP')
+    parser = argparse.ArgumentParser(description='Download and organize conversation files by annotator name')
     parser.add_argument('--sandbox-check', action='store_true', help='Check for matching sandbox logs')
     parser.add_argument('--download', action='store_true', help='Enable file download')
     args = parser.parse_args()
     
     # Download files if enabled
     if args.download:
-        print("\nDownloading files and organizing by IP address...")
-        download_files_by_ip(reader, check_sandbox=args.sandbox_check)
+        print("\nDownloading files and organizing by annotator name...")
+        download_files_by_name(reader, check_sandbox=args.sandbox_check)
     
     # Count and display statistics
-    ip_counts = count_files_per_ip(reader)
-    print("\nFile counts per IP address:")
-    for ip, count in sorted(ip_counts.items(), key=lambda x: x[1], reverse=True):
-        print(f"IP: {ip:<15} Count: {count}")
+    name_counts = count_files_per_ip(reader)
+    print("\nFile counts per annotator:")
+    for name, count in sorted(name_counts.items(), key=lambda x: x[1], reverse=True):
+        print(f"Name: {name:<20} Count: {count}")
 
 if __name__ == "__main__":
     main()