1
import sys, copy
2
sys.path.insert(0, '.')
3
sys.path.insert(0, '/path/to/submission')
4
from aiops_world import load_jsonl, world_state, rollout_policy
5
from aiops_world import observe, init_runtime, apply_action, task_public_view
6
from agent import Agent
7
from collections import Counter, defaultdict
8

9
vtasks = load_jsonl('valid_tasks.jsonl')
10
ptasks = load_jsonl('public_tasks.jsonl')
11
all_tasks = vtasks + ptasks
12

13
DEP_SVCS = {'svc_05', 'svc_06', 'svc_07', 'svc_08'}
14

15
total_oracle = total_correct = total_late = total_wrong = total_missing = 0
16
probe_total = dep_probe_total = 0
17
wrong_by_type = Counter()
18

19
for task in all_tasks:
20
    oracle_r = rollout_policy(task, 'oracle')
21
    oracle_acts_timed = {
22
        (a['type'], a.get('target', '')): a['time']
23
        for a in oracle_r['actions'] if a['type'] != 'NOOP'
24
    }
25
    oracle_acts = set(oracle_acts_timed.keys())
26
    total_oracle += len(oracle_acts)
27

28
    agent = Agent()
29
    agent.reset(copy.deepcopy(task_public_view(task)))
30
    runtime = init_runtime(task)
31
    agent_act_times = defaultdict(list)
32

33
    for t in range(48):
34
        obs = observe(task, runtime, t)
35
        action = agent.act(copy.deepcopy(obs))
36
        atype = action.get('type', 'NOOP')
37
        atgt = action.get('target', '')
38
        if atype == 'DIAGNOSTIC_PROBE':
39
            probe_total += 1
40
            if atgt in DEP_SVCS:
41
                dep_probe_total += 1
42
        elif atype != 'NOOP':
43
            agent_act_times[(atype, atgt)].append(t)
44
        apply_action(task, runtime, action, t)
45

46
    agent_acts = set(agent_act_times.keys())
47
    for key, times in agent_act_times.items():
48
        if key in oracle_acts:
49
            delay = times[0] - oracle_acts_timed[key]
50
            if delay <= 3:
51
                total_correct += 1
52
            else:
53
                total_late += 1
54
        else:
55
            total_wrong += 1
56
            wrong_by_type[f"{key[0]}({key[1]})"] += 1
57
    total_missing += len(oracle_acts - agent_acts)
58

59
print(f"Oracle: {total_oracle} actions")
60
print(f"Correct (≤3 steps): {total_correct} ({100*total_correct/total_oracle:.1f}%)")
61
print(f"Late (>3 steps): {total_late} ({100*total_late/total_oracle:.1f}%)")
62
print(f"Missing: {total_missing} ({100*total_missing/total_oracle:.1f}%)")
63
print(f"Wrong (not in oracle): {total_wrong}")
64
print(f"Probes: {probe_total} total, dep={dep_probe_total} ({100*dep_probe_total/probe_total:.1f}%)")
65
print("\nTop wrong actions:")
66
for k, v in wrong_by_type.most_common(10):
67
    print(f"  {v}x {k}")

1
import sys
2
sys.path.insert(0, '.')
3
from aiops_world import load_jsonl, world_state, rollout_policy
4
from collections import defaultdict
5

6
vtasks = load_jsonl('valid_tasks.jsonl')
7
ptasks = load_jsonl('public_tasks.jsonl')
8
all_tasks = vtasks + ptasks
9

10
SECONDARY_CANDIDATES = {
11
    "queue_drain":     "rollback_release",
12
    "cache_warm":      "timeout_circuit",
13
    "rollback_config": "restart_scale",
14
    "restart_scale":   "rollback_config",
15
    "scale_throttle":  "rollback_release",
16
    "rollback_release":"queue_drain",
17
    "storage_index":   "rollback_release",
18
}
19

20
HINT_TO_MODE = {
21
    'rollback_release': 'release_instability',
22
    'rollback_config': 'config_shift',
23
    'restart_scale': 'heap_growth',
24
    'scale_throttle': 'traffic_surge',
25
    'storage_index': 'storage_wait',
26
    'queue_drain': 'consumer_lag',
27
    'cache_warm': 'cache_collapse',
28
    'timeout_circuit': 'dependency_timeout',
29
}
30

31
print(f"{'Primary hint':22} {'Secondary':22} {'有效':8} {'无效':8} {'准确率':10}")
32
print("-" * 75)
33

34
for primary_hint, sec_hint in SECONDARY_CANDIDATES.items():
35
    primary_mode = HINT_TO_MODE.get(primary_hint, '')
36
    sec_mode = HINT_TO_MODE.get(sec_hint, '')
37
    benefit = harm = 0
38

39
    for task in all_tasks:
40
        world = world_state(task)
41
        svc_modes = defaultdict(list)
42
        for m in world['modes']:
43
            svc_modes[m['service']].append(m['mode'])
44

45
        for svc, modes in svc_modes.items():
46
            if primary_mode not in modes or len(modes) < 2:
47
                continue
48
            if sec_mode in modes:
49
                benefit += 1
50
            else:
51
                harm += 1
52

53
    total = benefit + harm
54
    precision = f"{100*benefit/total:.0f}%" if total > 0 else "N/A"
55
    print(f"{primary_hint:22} {sec_hint:22} {benefit:8} {harm:8} {precision:10}")

1
import sys
2
sys.path.insert(0, '.')
3
from aiops_world import load_jsonl, world_state, make_probe_result
4

5
vtasks = load_jsonl('valid_tasks.jsonl')
6
ptasks = load_jsonl('public_tasks.jsonl')
7
all_tasks = vtasks + ptasks
8

9
DEP_VALID_HINTS = {
10
    "svc_05": {"storage_index", "timeout_circuit"},
11
    "svc_06": {"storage_index", "timeout_circuit"},
12
    "svc_07": {"cache_warm", "timeout_circuit"},
13
    "svc_08": {"queue_drain", "timeout_circuit"},
14
}
15

16
filtered_noise = 0
17
missed_valid = 0
18

19
for task in all_tasks:
20
    world = world_state(task)
21
    for mode in world['modes']:
22
        dep = mode.get('dependency', '')
23
        if dep not in DEP_VALID_HINTS:
24
            continue
25
        svc = mode['service']
26
        t = mode['start'] + 3
27
        pr = make_probe_result(task, dep, t)
28
        hint = pr.get('action_hint', '')
29
        pressure = pr.get('pressure_bucket', 0)
30

31
        if pressure < 2 or hint in ('investigate',):
32
            continue
33

34
        if hint not in DEP_VALID_HINTS[dep]:
35
            filtered_noise += 1
36
        else:
37
            missed_valid += 0
38

39
noise_returns = 0
40
valid_returns = 0
41
for task in all_tasks:
42
    world = world_state(task)
43
    for dep_svc in DEP_VALID_HINTS:
44
        for t in range(5, 48, 5):
45
            pr = make_probe_result(task, dep_svc, t)
46
            hint = pr.get('action_hint', '')
47
            if pr.get('pressure_bucket', 0) < 2:
48
                continue
49
            if hint in DEP_VALID_HINTS[dep_svc]:
50
                valid_returns += 1
51
            else:
52
                noise_returns += 1
53
            break
54

55
print(f"Dep probe returns: {valid_returns} valid signal, {noise_returns} noise")
56
print(f"Noise ratio: {100*noise_returns/(valid_returns+noise_returns):.1f}%")
57
print(f"DEP_VALID_HINTS filters out: {noise_returns} noise returns")

1
import sys
2
sys.path.insert(0, '.')
3
from aiops_world import load_jsonl, world_state, make_probe_result, rollout_policy
4
from collections import defaultdict
5

6
vtasks = load_jsonl('valid_tasks.jsonl')
7
ptasks = load_jsonl('public_tasks.jsonl')
8
all_tasks = vtasks + ptasks
9

10
SVC_DEPS = {
11
    "svc_00": ["svc_07"], "svc_01": ["svc_07"],
12
    "svc_02": ["svc_06", "svc_07"],
13
    "svc_03": ["svc_05", "svc_08"],
14
    "svc_04": ["svc_05"],
15
    "svc_09": ["svc_05", "svc_08"],
16
}
17
PRIMARY_MODES = {'release_instability', 'config_shift', 'heap_growth', 'traffic_surge'}
18

19
ri_correct = ri_wrong = 0
20
st_correct = st_wrong = 0
21

22
for task in all_tasks:
23
    world = world_state(task)
24
    oracle_r = rollout_policy(task, 'oracle')
25
    oracle_acts = set((a['type'], a.get('target', '')) for a in oracle_r['actions'])
26

27
    svc_modes = defaultdict(list)
28
    for m in world['modes']:
29
        svc_modes[m['service']].append(m['mode'])
30

31
    for svc, modes in svc_modes.items():
32
        if not any(m in PRIMARY_MODES for m in modes):
33
            continue
34
        for dep in [d for d in SVC_DEPS.get(svc, []) if d in ('svc_05', 'svc_06')]:
35
            pm_start = next(
36
                (m['start'] for m in world['modes']
37
                 if m['mode'] in PRIMARY_MODES and m['service'] == svc), 48
38
            )
39
            pr = make_probe_result(task, dep, pm_start + 5)
40
            if pr.get('action_hint') == 'storage_index' and pr.get('pressure_bucket', 0) >= 2:
41
                if ('REBUILD_INDEX', dep) in oracle_acts:
42
                    ri_correct += 1
43
                else:
44
                    ri_wrong += 1
45
                if ('SET_TIMEOUT', svc) in oracle_acts:
46
                    st_correct += 1
47
                else:
48
                    st_wrong += 1
49

50
print(f"REBUILD_INDEX from dep probe: {ri_correct} correct, {ri_wrong} wrong "
51
      f"({100*ri_correct/(ri_correct+ri_wrong):.0f}%)")
52
print(f"SET_TIMEOUT(prim_svc) from dep probe: {st_correct} correct, {st_wrong} wrong "
53
      f"({100*st_correct/(st_correct+st_wrong):.0f}%)")

1
import sys
2
sys.path.insert(0, '.')
3
from aiops_world import load_jsonl, world_state, make_probe_result, rollout_policy
4

5
vtasks = load_jsonl('valid_tasks.jsonl')
6
ptasks = load_jsonl('public_tasks.jsonl')
7
all_tasks = vtasks + ptasks
8

9
HINT_TO_ACTIONS = {
10
    'rollback_release': lambda svc, src: [('ROLLBACK_RELEASE', svc), ('CIRCUIT_BREAK', 'svc_00')],
11
    'rollback_config':  lambda svc, src: [('ROLLBACK_CONFIG', svc), ('SET_TIMEOUT', svc)],
12
    'restart_scale':    lambda svc, src: [('RESTART', svc), ('SCALE', svc)],
13
    'scale_throttle':   lambda svc, src: [('SCALE', 'svc_00'), ('SCALE', svc)],
14
    'storage_index':    lambda svc, src: [('REBUILD_INDEX', src if src != svc else svc), ('SET_TIMEOUT', svc)],
15
    'queue_drain':      lambda svc, src: [('DRAIN_QUEUE', 'svc_08'), ('SCALE', 'svc_09')],
16
    'cache_warm':       lambda svc, src: [('WARM_CACHE', 'svc_07'), ('THROTTLE', 'svc_00')],
17
    'timeout_circuit':  lambda svc, src: [('CIRCUIT_BREAK', svc), ('SET_TIMEOUT', src if src != svc else svc)],
18
}
19

20
total_oracle = covered = 0
21
for task in all_tasks:
22
    world = world_state(task)
23
    oracle_r = rollout_policy(task, 'oracle')
24
    oracle_acts = set((a['type'], a.get('target', '')) for a in oracle_r['actions'] if a['type'] != 'NOOP')
25
    total_oracle += len(oracle_acts)
26

27
    predicted = set()
28
    for mode in world['modes']:
29
        svc = mode['service']
30
        t = mode['start'] + 3
31
        pr = make_probe_result(task, svc, t)
32
        hint = pr.get('action_hint', '')
33
        src = pr.get('source_hint', svc)
34
        if pr.get('pressure_bucket', 0) < 1 or hint not in HINT_TO_ACTIONS:
35
            continue
36
        for atype, atgt in HINT_TO_ACTIONS[hint](svc, src):
37
            predicted.add((atype, atgt))
38

39
    covered += len(oracle_acts & predicted)
40

41
print(f"{covered}/{total_oracle} = {100*covered/total_oracle:.1f}%")