#!/usr/bin/env python3
"""
Script to analyze JSONL data from stdin and count values per item.
Detects duplicate itemids and reports top 100 items with most values.
Includes data size calculations based on value_type.
"""

import json
import sys
import argparse
from collections import defaultdict, Counter
from typing import Dict, List, Set, Tuple
from datetime import datetime, timedelta


def calculate_value_size(value_type: int, value_str: str = None) -> int:
    """
    Calculate the size of a single value based on value_type.
    
    Args:
        value_type: The type of the value (1, 2, 3, or 6)
        value_str: The string representation of the value (for types 1 and 6)
    
    Returns:
        Size in bytes
    """
    if value_type in [2, 3]:
        return 20
    elif value_type in [1, 6]:
        # 40 bytes base + string length
        base_size = 40
        string_size = len(value_str) if value_str else 0
        return base_size + string_size
    else:
        # Default size for unknown types
        return 20


def analyze_jsonl_data(encoding='utf-8'):
    """
    Read JSONL data from stdin and analyze item values.
    
    Args:
        encoding: The encoding to use for reading input (default: utf-8)
    """
    item_value_counts: Dict[int, int] = {}
    item_timestamps: Dict[int, Tuple[int, int]] = {}  # (min_timestamp, max_timestamp)
    item_data_sizes: Dict[int, int] = {}  # Total data size per item
    seen_itemids: Set[int] = set()
    duplicate_itemids: Set[int] = set()
    line_count = 0
    total_data_size = 0
    
    print("Reading JSONL data from stdin...", file=sys.stderr)
    
    try:
        for line_num, line in enumerate(sys.stdin.buffer, 1):
            try:
                # Use specified encoding, with fallback to latin-1
                line = line.decode(encoding).strip()
            except UnicodeDecodeError:
                if encoding != 'latin-1':
                    try:
                        line = line.decode('latin-1').strip()
                        print(f"Warning: Line {line_num} not valid {encoding}, using latin-1", file=sys.stderr)
                    except UnicodeDecodeError:
                        print(f"Warning: Line {line_num} has invalid encoding, skipping", file=sys.stderr)
                        continue
                else:
                    print(f"Warning: Line {line_num} has invalid encoding, skipping", file=sys.stderr)
                    continue
            
            if not line:
                continue
                
            # Progress notification every 100k lines
            if line_num % 100000 == 0:
                print(f"Progress: Processed {line_num:,} lines...", file=sys.stderr)
                
            try:
                data = json.loads(line)
                itemid = data.get('itemid')
                value_type = data.get('value_type')
                
                if itemid is None:
                    print(f"Warning: Line {line_num} missing 'itemid' field", file=sys.stderr)
                    continue
                
                # Check for duplicate itemids
                if itemid in seen_itemids:
                    duplicate_itemids.add(itemid)
                    print(f"Warning: Duplicate itemid {itemid} found at line {line_num}", file=sys.stderr)
                else:
                    seen_itemids.add(itemid)
                
                # Count values and track timestamps
                values = data.get('values', [])
                value_count = len(values)
                item_value_counts[itemid] = value_count
                
                # Track min and max timestamps
                if values:
                    timestamps = [v.get('clock', 0) for v in values if v.get('clock') is not None]
                    if timestamps:
                        min_ts = min(timestamps)
                        max_ts = max(timestamps)
                        item_timestamps[itemid] = (min_ts, max_ts)
                
                # Calculate data size for this item
                item_size = 0
                if value_type is not None and values:
                    for value_obj in values:
                        value = value_obj.get('value')
                        if value is not None:
                            # Convert value to string for size calculation
                            value_str = str(value)
                            item_size += calculate_value_size(value_type, value_str)
                
                item_data_sizes[itemid] = item_size
                total_data_size += item_size
                
                line_count += 1
                
            except json.JSONDecodeError as e:
                print(f"Error: Invalid JSON at line {line_num}: {e}", file=sys.stderr)
                continue
            except Exception as e:
                print(f"Error: Unexpected error at line {line_num}: {e}", file=sys.stderr)
                continue
    
    except KeyboardInterrupt:
        print("\nInterrupted by user", file=sys.stderr)
        return
    
    # Report summary
    print(f"\nAnalysis Summary:", file=sys.stderr)
    print(f"Total items processed: {len(item_value_counts)}", file=sys.stderr)
    print(f"Total lines processed: {line_count}", file=sys.stderr)
    print(f"Total data size: {total_data_size:,} bytes ({total_data_size / 1024 / 1024:.2f} MB)", file=sys.stderr)
    
    if duplicate_itemids:
        print(f"Duplicate itemids found: {len(duplicate_itemids)}", file=sys.stderr)
        print(f"Duplicate itemids: {sorted(duplicate_itemids)}", file=sys.stderr)
    else:
        print("No duplicate itemids found", file=sys.stderr)
    
    # Find top 100 items with most values
    sorted_items = sorted(item_value_counts.items(), key=lambda x: x[1], reverse=True)
    top_100 = sorted_items[:100]
    
    print(f"\nTop 100 items with most values:", file=sys.stderr)
    print(f"{'Rank':<6} {'ItemID':<15} {'Value Count':<12} {'Data Size':<12} {'Time Range':<25} {'Duration':<15}")
    print("-" * 95)
    
    for rank, (itemid, count) in enumerate(top_100, 1):
        time_range_str = "N/A"
        duration_str = "N/A"
        data_size = item_data_sizes.get(itemid, 0)
        
        if itemid in item_timestamps:
            min_ts, max_ts = item_timestamps[itemid]
            min_dt = datetime.fromtimestamp(min_ts)
            max_dt = datetime.fromtimestamp(max_ts)
            time_range_str = f"{min_dt.strftime('%Y-%m-%d %H:%M:%S')} - {max_dt.strftime('%Y-%m-%d %H:%M:%S')}"
            
            duration = max_ts - min_ts
            if duration > 0:
                duration_str = str(timedelta(seconds=duration))
            else:
                duration_str = "0s"
        
        print(f"{rank:<6} {itemid:<15} {count:<12} {data_size:<12} {time_range_str:<25} {duration_str:<15}")
    
    # Find top 100 items with longest duration
    items_with_duration = []
    for itemid, (min_ts, max_ts) in item_timestamps.items():
        duration = max_ts - min_ts
        if duration > 0:  # Only include items with valid duration
            items_with_duration.append((itemid, duration))
    
    # Sort by duration (descending)
    items_with_duration.sort(key=lambda x: x[1], reverse=True)
    top_100_duration = items_with_duration[:100]
    
    print(f"\nTop 100 items with longest duration:", file=sys.stderr)
    print(f"{'Rank':<6} {'ItemID':<15} {'Duration':<15} {'Duration (s)':<12} {'Value Count':<12} {'Data Size':<12} {'Time Range':<25}")
    print("-" * 100)
    
    for rank, (itemid, duration) in enumerate(top_100_duration, 1):
        count = item_value_counts.get(itemid, 0)
        data_size = item_data_sizes.get(itemid, 0)
        min_ts, max_ts = item_timestamps[itemid]
        min_dt = datetime.fromtimestamp(min_ts)
        max_dt = datetime.fromtimestamp(max_ts)
        time_range_str = f"{min_dt.strftime('%Y-%m-%d %H:%M:%S')} - {max_dt.strftime('%Y-%m-%d %H:%M:%S')}"
        duration_str = str(timedelta(seconds=duration))
        
        print(f"{rank:<6} {itemid:<15} {duration_str:<15} {duration:<12} {count:<12} {data_size:<12} {time_range_str:<25}")
    
    # Find top 100 items with largest data size
    items_with_size = [(itemid, size) for itemid, size in item_data_sizes.items() if size > 0]
    items_with_size.sort(key=lambda x: x[1], reverse=True)
    top_100_size = items_with_size[:100]
    
    print(f"\nTop 100 items with largest data size:", file=sys.stderr)
    print(f"{'Rank':<6} {'ItemID':<15} {'Data Size':<12} {'Data Size (MB)':<15} {'Value Count':<12} {'Time Range':<25}")
    print("-" * 95)
    
    for rank, (itemid, size) in enumerate(top_100_size, 1):
        count = item_value_counts.get(itemid, 0)
        size_mb = size / 1024 / 1024
        time_range_str = "N/A"
        
        if itemid in item_timestamps:
            min_ts, max_ts = item_timestamps[itemid]
            min_dt = datetime.fromtimestamp(min_ts)
            max_dt = datetime.fromtimestamp(max_ts)
            time_range_str = f"{min_dt.strftime('%Y-%m-%d %H:%M:%S')} - {max_dt.strftime('%Y-%m-%d %H:%M:%S')}"
        
        print(f"{rank:<6} {itemid:<15} {size:<12} {size_mb:<15.2f} {count:<12} {time_range_str:<25}")
    
    # Also output as JSON for programmatic use
    print("\nJSON output:")
    result = {
        "summary": {
            "total_items": len(item_value_counts),
            "total_lines": line_count,
            "total_data_size_bytes": total_data_size,
            "total_data_size_mb": total_data_size / 1024 / 1024,
            "duplicate_itemids": list(duplicate_itemids),
            "duplicate_count": len(duplicate_itemids)
        },
        "top_100_items_by_value_count": [
            {
                "rank": rank, 
                "itemid": itemid, 
                "value_count": count,
                "data_size_bytes": item_data_sizes.get(itemid, 0),
                "data_size_mb": item_data_sizes.get(itemid, 0) / 1024 / 1024,
                "time_range": {
                    "min_timestamp": item_timestamps.get(itemid, (None, None))[0],
                    "max_timestamp": item_timestamps.get(itemid, (None, None))[1],
                    "min_datetime": datetime.fromtimestamp(item_timestamps[itemid][0]).isoformat() if itemid in item_timestamps else None,
                    "max_datetime": datetime.fromtimestamp(item_timestamps[itemid][1]).isoformat() if itemid in item_timestamps else None,
                    "duration_seconds": item_timestamps[itemid][1] - item_timestamps[itemid][0] if itemid in item_timestamps else None,
                    "duration_human": str(timedelta(seconds=item_timestamps[itemid][1] - item_timestamps[itemid][0])) if itemid in item_timestamps and item_timestamps[itemid][1] > item_timestamps[itemid][0] else "0s"
                }
            }
            for rank, (itemid, count) in enumerate(top_100, 1)
        ],
        "top_100_items_by_duration": [
            {
                "rank": rank,
                "itemid": itemid,
                "duration_seconds": duration,
                "duration_human": str(timedelta(seconds=duration)),
                "value_count": item_value_counts.get(itemid, 0),
                "data_size_bytes": item_data_sizes.get(itemid, 0),
                "data_size_mb": item_data_sizes.get(itemid, 0) / 1024 / 1024,
                "time_range": {
                    "min_timestamp": item_timestamps[itemid][0],
                    "max_timestamp": item_timestamps[itemid][1],
                    "min_datetime": datetime.fromtimestamp(item_timestamps[itemid][0]).isoformat(),
                    "max_datetime": datetime.fromtimestamp(item_timestamps[itemid][1]).isoformat()
                }
            }
            for rank, (itemid, duration) in enumerate(top_100_duration, 1)
        ],
        "top_100_items_by_data_size": [
            {
                "rank": rank,
                "itemid": itemid,
                "data_size_bytes": size,
                "data_size_mb": size / 1024 / 1024,
                "value_count": item_value_counts.get(itemid, 0),
                "time_range": {
                    "min_timestamp": item_timestamps.get(itemid, (None, None))[0],
                    "max_timestamp": item_timestamps.get(itemid, (None, None))[1],
                    "min_datetime": datetime.fromtimestamp(item_timestamps[itemid][0]).isoformat() if itemid in item_timestamps else None,
                    "max_datetime": datetime.fromtimestamp(item_timestamps[itemid][1]).isoformat() if itemid in item_timestamps else None,
                    "duration_seconds": item_timestamps[itemid][1] - item_timestamps[itemid][0] if itemid in item_timestamps else None,
                    "duration_human": str(timedelta(seconds=item_timestamps[itemid][1] - item_timestamps[itemid][0])) if itemid in item_timestamps and item_timestamps[itemid][1] > item_timestamps[itemid][0] else "0s"
                }
            }
            for rank, (itemid, size) in enumerate(top_100_size, 1)
        ]
    }
    print(json.dumps(result, indent=2))


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Analyze JSONL data from stdin')
    parser.add_argument('--encoding', '-e', default='utf-8', 
                       help='Input encoding (default: utf-8, common alternatives: latin-1, cp1252)')
    parser.add_argument('--no-fallback', action='store_true',
                       help='Disable fallback to latin-1 encoding')
    
    args = parser.parse_args()
    
    # If no-fallback is specified, use the exact encoding without fallback
    if args.no_fallback:
        analyze_jsonl_data(args.encoding)
    else:
        analyze_jsonl_data(args.encoding)