
    "Yjؖ                    ^   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZmZmZmZmZ ddlmZ ddlmZ ddlmZ  ej        e          ZeZdad	ed
<    ej                    Zdaded<    ej                    Z i Z!ded<   dZ"dZ#dZ$dZ% ej                    Z&d Z'dZdZ(d[dZ)d\dZ*d]dZ+d^d!Z,d\d"Z-d_d#Z.d_d$Z/d`d&Z0dad(Z1dbd,Z2dad-Z3dad.Z4dcd/Z5dcd0Z6ddd2Z7ded5Z8d_d6Z9dfd7Z:d]d8Z;dd9de"d:dgdIZ<dhdKZ=didLZ>dd9de"d:djdOZ?dkdQZ@dldSZAdmdndVZB	 	 	 	 dodpdXZCd]dYZDdS )qu  
Async (background) delegation registry.

Backs ``delegate_task(background=true)``: the parent agent dispatches a
subagent that runs on a module-level daemon executor and returns a handle
immediately, so the user and the model can keep working while the child runs.

When the child finishes, a completion event is pushed onto the SHARED
``process_registry.completion_queue`` with ``type="async_delegation"``. The
CLI (``cli.py`` process_loop) and gateway (``_run_process_watcher`` /
``completion_queue`` drain) already poll that queue while the agent is idle
and forge a fresh user/internal turn from each event. We deliberately reuse
that rail rather than reaching into a running agent loop:

  - completions surface as a NEW turn when the agent is idle, never spliced
    between a tool result and an assistant message. That keeps strict
    message-role alternation legal and the prompt cache intact (hard
    invariant: never mutate past context).
  - we inherit the queue's de-dup, crash-recovery checkpoint, and the
    existing CLI + gateway drain wiring for free — no new drain loops in the
    two largest files in the repo.

The completion payload carries a RICH, self-contained task-source block (the
original goal, the context the parent supplied, toolsets, model, dispatch
time, status, and the full result summary). When the result re-enters the
conversation the parent may be deep in unrelated context and won't remember
why the subagent existed; the block lets it either use the result or
re-dispatch if the world has moved on.

This module owns ONLY the async lifecycle. The actual child build + run is
delegated back to ``delegate_tool._run_single_child`` via an injected
runner, so all the credential leasing, heartbeat, timeout, and result-shaping
logic stays in one place.
    )annotationsN)ThreadPoolExecutor)AnyCallableDictListOptionalget_hermes_home)DaemonThreadPoolExecutor)propagate_context_to_threadzOptional[ThreadPoolExecutor]	_executorint_executor_max_workerszDict[str, Dict[str, Any]]_records   2   i:	 i  c                 $    t                      dz  S )Nzstate.dbr
        KC:\Users\server\AppData\Local\hermes\hermes-agent\tools\async_delegation.py_db_pathr   S   s    z))r   returnsqlite3.Connectionc                 f   t                      } | j                            dd           t          j        | d          }|                    d           |                    d           d |                    d          D             }d	D ]$\  }}||vr|                    d
| d|            %|S )NT)parentsexist_ok
   )timeoutzPRAGMA journal_mode=WALa  CREATE TABLE IF NOT EXISTS async_delegations (
            delegation_id TEXT PRIMARY KEY,
            origin_session TEXT NOT NULL,
            origin_ui_session_id TEXT NOT NULL DEFAULT '',
            parent_session_id TEXT,
            state TEXT NOT NULL,
            dispatched_at REAL NOT NULL,
            completed_at REAL,
            updated_at REAL NOT NULL,
            event_json TEXT,
            result_json TEXT,
            delivery_state TEXT NOT NULL DEFAULT 'pending',
            delivery_attempts INTEGER NOT NULL DEFAULT 0,
            delivered_at REAL,
            owner_pid INTEGER,
            owner_started_at INTEGER,
            task_json TEXT,
            delivery_claim TEXT,
            delivery_claimed_at REAL
        )c                    h | ]
}|d          S )   r   ).0rows     r   	<setcomp>z_connect.<locals>.<setcomp>r   s    VVV#s1vVVVr   z$PRAGMA table_info(async_delegations)))	owner_pidINTEGER)owner_started_atr&   )	task_jsonTEXT)delivery_claimr)   )delivery_claimed_atREALz)ALTER TABLE async_delegations ADD COLUMN  )r   parentmkdirsqlite3connectexecute)pathconncolumnsnamesql_types        r   _connectr8   W   s    ::DKdT222?4,,,DLL*+++LL	  , WV.T!U!UVVVG X Xh wLLVTVVHVVWWWKr   recordDict[str, Any]Nonec                    t          j                     }	 ddlm}  |t          d                                                    }n# t
          $ r d }Y nw xY w fddD             }t          5  t                      5 }|                    d d          	                    dd	           	                    d
d	           	                    d           d         |t          d                                          |t          j        |          f	           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t                       d S )Nr   )get_process_start_timeosc                D    i | ]}|v |                     |          S r   get)r"   keyr9   s     r   
<dictcomp>z%_persist_dispatch.<locals>.<dictcomp>   s6       &== 	VZZ__==r   )goalgoalscontexttoolsetsrolemodelis_batchah  INSERT OR REPLACE INTO async_delegations
               (delegation_id, origin_session, origin_ui_session_id,
                parent_session_id, state, dispatched_at, updated_at,
                delivery_state, delivery_attempts, owner_pid,
                owner_started_at, task_json)
               VALUES (?, ?, ?, ?, 'running', ?, ?, 'pending', 0, ?, ?, ?)delegation_idsession_key origin_ui_session_idparent_session_iddispatched_at)timegateway.statusr=   
__import__getpid	Exception_DB_LOCKr8   r2   rA   jsondumps_prune_durable_records)r9   nowr=   r'   task_payloadr4   s   `     r   _persist_dispatchr\      s   
)++C 99999911*T2B2B2I2I2K2KLL          X  L
 
 
 
8:: 
N O$fjj&C&CZZ.33VZZ@S5T5TO$c:d+;+;+B+B+D+Dtz,779	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 sH   0A AA.D8=BD!D8!D%	%D8(D%	)D88D<?D<rK   strc                    t           5  t                      5 }|                    d| f           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz3DELETE FROM async_delegations WHERE delegation_id=?)rV   r8   r2   rK   r4   s     r   _delete_durable_delegationr`      s   	 ^ ^8:: ^J]L\]]]^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^s.   A;A?	A?	AAAc                 r   t          j                     } | t          z
  }t          5  t                      5 }|                    d|f           |                    d                                          d         }t          d|t          z
            }|r|                    d|f           |                    d                                          d         }t          d|t          z
            }|r|                    d|f           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zBBound terminal history, preferring delivered records for deletion.zQDELETE FROM async_delegations WHERE delivery_state='delivered' AND updated_at < ?zRSELECT COUNT(*) FROM async_delegations WHERE state NOT IN ('running','finalizing')r   aX  DELETE FROM async_delegations WHERE delegation_id IN (
                     SELECT delegation_id FROM async_delegations
                     WHERE state NOT IN ('running','finalizing')
                     ORDER BY CASE delivery_state WHEN 'delivered' THEN 0 ELSE 1 END,
                              updated_at ASC LIMIT ?
                   )z~SELECT COUNT(*) FROM async_delegations
               WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'a  DELETE FROM async_delegations WHERE delegation_id IN (
                     SELECT delegation_id FROM async_delegations
                     WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'
                     ORDER BY updated_at ASC LIMIT ?
                   )N)	rQ   _DURABLE_RETENTION_SECONDSrV   r8   r2   fetchonemax_MAX_RETAINED_COMPLETED_MAX_DURABLE_PENDING)rZ   cutoffr4   terminal_countexcesspending_countoverflows          r   rY   rY      s   
)++C--F	    8::  _I	
 	
 	
 `
 

(**Q Q)@@AA 		LL 	   [
 
 (**Q q-*>>?? 	LL
   3                                                                 s5   D,CDD,D	D,D	D,,D03D0eventresultc                   t          j                     }t          5  t                      5 }|                    d|                     dd          |                     d|          |t          j        |           t          j        |          | d         f           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NzUPDATE async_delegations SET state=?, completed_at=?, updated_at=?,
               event_json=?, result_json=?, delivery_state='pending'
               WHERE delegation_id=?status	completedcompleted_atrK   )rQ   rV   r8   r2   rA   rW   rX   )rl   rm   rZ   r4   s       r   _persist_completionrr      s8   
)++C	 
 
8:: 
( YYx--uyy/M/MsZ
6 2 2E/4JL		
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s5   B=A/B%B=%B)	)B=,B)	-B==CCc                    t           5  t                      5 }|                    dt          j                    | f           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NzfUPDATE async_delegations SET delivery_attempts=delivery_attempts+1, updated_at=? WHERE delegation_id=?)rV   r8   r2   rQ   r_   s     r   _note_delivery_attemptrt      s    	 
 
8:: 
tY[[-(	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s4   A%*AA%A	A%A	A%%A),A)c                 X   	 ddl m} m} n# t          $ r Y dS w xY wt	          j                    }d}t
          5  t                      5 }|                    d                                          }|D ]}|\  }}}	}
}}}}d}|rE | t          |                    }|r+|) |t          |                    t          |          k    }|rZt          j        |pd          }i ddd	|d
|d|	d|
d|                    dd          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          dt          |                    d                    ddddddd|d|}dd|d         d}|                    d||t          j        |          t          j        |          |f           |dz  }	 ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   |S )zEClassify records whose owning process disappeared as outcome unknown.r   )_pid_existsr=   zSELECT delegation_id, origin_session, origin_ui_session_id,
                      parent_session_id, dispatched_at, owner_pid,
                      owner_started_at, task_json
               FROM async_delegations WHERE state IN ('running','finalizing')FNz{}typeasync_delegationrK   rL   rN   rO   rD   rM   rE   rF   rG   rH   rI   rJ   ro   unknownsummaryerrorzLDelegation owner exited before recording a terminal result; outcome unknown.rP   rq   )ro   rz   r{   zUPDATE async_delegations SET state='unknown', completed_at=?,
                   updated_at=?, event_json=?, result_json=?, delivery_state='pending'
                   WHERE delegation_id=?r!   )rR   rv   r=   rU   rQ   rV   r8   r2   fetchallr   rW   loadsrA   boolrX   )rv   r=   rZ   	recoveredr4   rowsr#   rK   rL   	origin_ui	parent_idrP   pidstartedr(   livetaskrl   rm   s                      r   recover_abandoned_delegationsr      se   FFFFFFFFF   qq
)++CI	 # #8:: #||Q
 

 (** 	  	 	CgjdM;	9mSRY[dD L"{3s88,, LG/11#c((;;s7||KD :i/400D
*
,;]
{
,BI
 $Y
 178L8L
 '**	
 -6txx	7J7J	

 DHHZ00

 39$((6:J:J
 '**
 -7TXXj=Q=Q8R8R
 )
 &/
 g
  
 1?
E !*dU7^TTFLL, c4:e,,dj.@.@-P	   NII9	# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #H sB    
HF4H<HH	HH	HH#&H#c                   t                       t          5  t                      5 }|                    d                                          }|D ]H\  }}t          j        |          }t          |t                    rd|d<   | 	                    |           I	 ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   t          |          S )uz  Enqueue durable pending completions as fresh turns after process start.

    Every restored event is stamped ``restored=True`` (in-memory only — the
    stamp is added after the durable payload is deserialized and is never
    persisted). Restored events originate from a *previous* process, so no
    consumer in THIS process implicitly owns them: drain paths that run
    without an ownership filter (the legacy single-session behavior) must
    leave them queued for a consumer that can positively prove ownership,
    otherwise a brand-new session adopts a dead session's delegation
    results seconds after boot (#64484).
    zSELECT delegation_id, event_json FROM async_delegations
               WHERE state != 'running' AND delivery_state='pending' AND event_json IS NOT NULL
               ORDER BY completed_at, delegation_idTrestoredN)r   rV   r8   r2   r|   rW   r}   
isinstancedictputlen)target_queuer4   r   _delegation_idpayloadevts         r   restore_undelivered_completionsr   
  sV    "###	 
" 
"8:: 
"||7
 
 (**	 	
 (, 	" 	"#NG*W%%C#t$$ '"&JS!!!!		"
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" t99s5   B<A3B%B<%B)	)B<,B)	-B<<C C r~   c                   t          j                     }t          5  t                      5 }|                    d||| f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zDAtomically acknowledge successful injection of a durable completion.zUPDATE async_delegations SET delivery_state='delivered', delivered_at=?, updated_at=?
               WHERE delegation_id=? AND delivery_state!='delivered'r!   NrQ   rV   r8   r2   rowcount)rK   rZ   r4   curs       r   mark_completion_deliveredr   %  s)   
)++C	 ! !8:: !llH#}%
 

 |q ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s4   A?$A'A?'A+	+A?.A+	/A??BBclaim_idc                   t          j                     }t          5  t                      5 }|                    d| f                                          }|	 ddd           ddd           dS |                    d|||| |dz
  f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zBClaim one pending completion across competing consumers/processes.zBSELECT delivery_state FROM async_delegations WHERE delegation_id=?NTa  UPDATE async_delegations SET delivery_claim=?, delivery_claimed_at=?,
                      delivery_attempts=delivery_attempts+1, updated_at=?
               WHERE delegation_id=? AND delivery_state='pending'
                 AND (delivery_claim IS NULL OR delivery_claimed_at < ?)i,  r!   )rQ   rV   r8   r2   rc   r   )rK   r   rZ   r4   r#   r   s         r   claim_completion_deliveryr   1  s   
)++C	 ! !8:: !llP
 
 (** 	 ;! ! ! ! ! ! ! ! ! ! ! ! ! ! llL sCc	:
 
 |q ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s@   C-B0C/(B0C0B4	4C7B4	8CCCr   consumerOptional[str]c                .   |                      d          dk    rdS t          |                      d          pd          }|sdS | dt          d                                           dt	          j                    j         }t          ||          r|ndS )zCClaim a durable delegation event; non-durable events need no token.rw   rx   rM   rK   :r>   N)rA   r]   rS   rT   uuiduuid4hexr   )r   r   rK   r   s       r   claim_event_deliveryr   E  s    
wwv,,,r006B77M rKKZ--4466KK9IKKH0IIS88tSr   c                   t           5  t                      5 }|                    dt          j                    | |f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z>Release a failed delivery claim so another consumer may retry.zUPDATE async_delegations SET delivery_claim=NULL,
                      delivery_claimed_at=NULL, updated_at=?
               WHERE delegation_id=? AND delivery_state='pending'
                 AND delivery_claim=?r!   N)rV   r8   r2   rQ   r   )rK   r   r4   r   s       r   release_completion_deliveryr   P  s%   	 ! !8:: !ll) Y[[-2
 
 |q ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s4   A=5A%A=%A)	)A=,A)	-A==BBc           	        t          j                     }t          5  t                      5 }|                    d||| |f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z;Acknowledge acceptance for the consumer holding this claim.a  UPDATE async_delegations SET delivery_state='delivered',
                      delivered_at=?, updated_at=?, delivery_claim=NULL,
                      delivery_claimed_at=NULL
               WHERE delegation_id=? AND delivery_state='pending'
                 AND delivery_claim=?r!   Nr   )rK   r   rZ   r4   r   s        r   complete_completion_deliveryr   ]  s,   
)++C	 	! 	!8:: 	!ll)
 #}h/
 
 |q 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!s4   B %A(B (A,	,B /A,	0B  BBc                    |rM|                      d          dk    r6t          t          |                      d          pd          |           d S d S d S Nrw   rx   rK   rM   )rA   r   r]   r   r   s     r   complete_event_deliveryr   l  sa     TCGGFOO'999$S)A)A)GR%H%H(SSSSST T99r   c                    |rM|                      d          dk    r6t          t          |                      d          pd          |           d S d S d S r   )rA   r   r]   r   s     r   release_event_deliveryr   q  sa     SCGGFOO'999#C(@(@(FB$G$GRRRRRS S99r   Optional[Dict[str, Any]]c           	        t           5  t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |d S | |d         |d         |d         |d         |d         rt	          j        |d                   nd |d         |d         d	S )
NzSELECT origin_session, state, dispatched_at, completed_at,
                      result_json, delivery_state, delivery_attempts
               FROM async_delegations WHERE delegation_id=?r   r!      r            )rK   origin_sessionstaterP   rq   rm   delivery_statedelivery_attempts)rV   r8   r2   rc   rW   r}   )rK   r4   r#   s      r   get_durable_delegationr   v  sG   	  8:: ll?AN@P
 
 (**	 	                              {t&#a&3q6QQ(+A8$*SV$$$Da&s1v	  s4   A$*AA$A	A$A	A$$A(+A(max_workersr   c                    t           5  t          | t          k    rt          | d          a| at          cddd           S # 1 swxY w Y   dS )u  Lazily create (or grow) the shared daemon executor.

    We never shrink — ThreadPoolExecutor can't resize — but if the configured
    cap grows between calls we rebuild a larger pool. Existing in-flight
    futures keep running on the old pool until it's garbage collected.
    Nzasync-delegate)r   thread_name_prefix)_executor_lockr   r   _DaemonThreadPoolExecutor)r   s    r   _get_executorr     s     
  .C C C1'#3  I %0!                 s   ,AAAc                     t           5  t          d t                                          D                       cddd           S # 1 swxY w Y   dS )z.Number of async delegations currently running.c              3  H   K   | ]}|                     d           dv dV  dS )ro   >   running
finalizingr!   Nr@   r"   rs     r   	<genexpr>zactive_count.<locals>.<genexpr>  s7      ``155??F_3_3_13_3_3_3_``r   N)_records_locksumr   valuesr   r   r   active_countr     s    	 a a``hoo//`````a a a a a a a a a a a a a a a a a as   0AA	A	c                 H    dt          j                    j        d d          S )Ndeleg_   )r   r   r   r   r   r   _new_delegation_idr     s"    *DJLL$RaR(***r   c                 .   d t                                           D             } t          |           t          k    rdS |                     d            | dt          |           t          z
           D ] \  }}t                               |d           !dS )ziDrop the oldest completed records beyond the retention cap.

    Caller must hold ``_records_lock``.
    c                N    g | ]"\  }}|                     d           dk    ||f#S ro   r   r@   )r"   ridr   s      r   
<listcomp>z+_prune_completed_locked.<locals>.<listcomp>  s?       C55??i'' 
a'''r   Nc                r    | d                              d          p| d                              d          pdS )Nr!   rq   rP   r   r@   )kvs    r   <lambda>z)_prune_completed_locked.<locals>.<lambda>  s1    "Q%))N";";"^r!uyy?Y?Y"^]^ r   )rB   )r   itemsr   re   sortpop)rp   r   _s      r   _prune_completed_lockedr     s    
 nn&&  I
 9~~000NN^^N___Fc)nn/FFFG    QS$   r   rM   )rO   rN   interrupt_fnmax_async_childrenrD   rF   rG   Optional[List[str]]rH   rI   rL   rO   runnerCallable[[], Dict[str, Any]]rN   r   Optional[Callable[[], None]]r   c                4   t                      t          j                    | ||rt          |          nd|||||dd|	d}t          5  t	          d t
                                          D                       }||
k    rdd|
 ddcddd           S |t
          <   ddd           n# 1 swxY w Y   t          |           t          |
          }dfd}	 |	                    t          |                     nh# t          $ r[}t          5  t
                              d           ddd           n# 1 swxY w Y   t                     dd| dcY d}~S d}~ww xY wt                              d|pd| pddd                    ddS )a  Spawn ``runner`` on the daemon executor and return a handle immediately.

    Parameters
    ----------
    goal, context, toolsets, role, model
        The dispatch-time task spec, captured verbatim for the rich
        completion block.
    session_key
        The gateway session_key (from ``tools.approval.get_current_session_key``)
        captured on the parent thread BEFORE dispatch, because the daemon
        worker thread won't carry the contextvar. Used to route the
        completion back to the originating session.
    parent_session_id
        The durable ``state.db`` session id of the parent agent that spawned
        the delegation. Carried on the completion event so the gateway can
        pin routing to the spawning session instead of recovering the latest
        ``ended_at IS NULL`` row for the peer tuple (#57498).
    runner
        Zero-arg callable that builds + runs the child and returns the same
        result dict ``_run_single_child`` produces. Runs on the worker thread.
    interrupt_fn
        Optional callable to signal the child to stop (used on shutdown /
        explicit cancel).
    max_async_children
        Concurrency cap. When at capacity the dispatch is REJECTED (the caller
        should fall back to sync or tell the user) rather than queued, so a
        runaway model can't pile up unbounded background work.

    Returns
    -------
    dict
        ``{"status": "dispatched", "delegation_id": ...}`` on success, or
        ``{"status": "rejected", "error": ...}`` when at capacity.
    Nr   )rK   rD   rF   rG   rH   rI   rL   rN   rO   ro   rP   rq   r   c              3  L   K   | ]}|                     d           dk    dV   dS ro   r   r!   Nr@   r   s     r   r   z,dispatch_async_delegation.<locals>.<genexpr>  ?       
 
AEE(OOy,H,HA,H,H,H,H
 
r   rejected#Async delegation capacity reached (z running). Wait for one to finish (its result will re-enter the chat), or run this task synchronously (background=false). Raise delegation.max_concurrent_children in config.yaml to allow more concurrent background subagents.ro   r{   r   r;   c            	        i } d}	              pi } |                      d          pd}nu# t          $ rh}t                              d           dd t	          |          j         d| dt          t          j                    z
  d          d} d}Y d }~nd }~ww xY wt          | |           d S # t          | |           w xY w)	Nr{   ro   rp   zAsync delegation %s crashed: r   r   )ro   rz   r{   	api_callsduration_seconds)	rA   rU   logger	exceptionrw   __name__roundrQ   	_finalize)rm   ro   excrK   rP   r   s      r   _workerz*dispatch_async_delegation.<locals>._worker  s    !#	5VXX^FZZ))8[FF 		 		 		:MJJJ! II.77#77$)$)++*Eq$I$I F FFFFFF		 mVV44444ImVV4444s,   #+ B3 
BABB3 BB3 3Cz%Failed to schedule async delegation: z3Dispatched async delegation %s (session_key=%s): %s<cli>rM   P   
dispatchedro   rK   r   r;   )r   rQ   listr   r   r   r   r\   r   submitr   rU   r   r`   r   info)rD   rF   rG   rH   rI   rL   rO   r   rN   r   r   r9   r   executorr   r   rK   rP   s          `        @@r   dispatch_async_delegationr     s   ` '((MIKKM&&.8DNNND" 4.&$ F$ 
 ) ) 
 
((
 
 
 
 
 ((($R:L R R R	 	) ) ) ) ) ) ) ) #)) ) ) ) ) ) ) ) ) ) ) ) ) ) )" f/00H5 5 5 5 5 5 5 5&
 	3G<<==== 
 
 
 	. 	.LL---	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	."=111 BSBB
 
 	
 	
 	
 	
 	
 	
	
 KK={-g
CRC/@   #]CCCsZ   ?B/
B//B36B3!"D 
E)E$D=1E$=E	E$E	E$E)$E)ro   c                   t           5  t                              |           }|	 ddd           dS d|d<   t          j                    |d<   d|d<   t	          |          }ddd           n# 1 swxY w Y   t          |||           t           5  t                              |           }|||d<   t                       ddd           dS # 1 swxY w Y   dS )zDMark a record complete and push the completion event onto the queue.Nr   ro   rq   r   )r   r   rA   rQ   r   _push_completion_eventr   )rK   rm   ro   r9   event_records        r   r   r   5  sz   	 
$ 
$m,,>
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ (x!%~!%~F||
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ <888	 " "m,,%F8!!!	" " " " " " " " " " " " " " " " " "s(   A./A..A25A20CCCc           	        	 ddl m} nG# t          $ r:}t                              d|                     d          |           Y d}~dS d}~ww xY w|                    d          }|                    d          }|                     d          pt          j                    }|                     d	          pt          j                    }i d
dd|                     d          d|                     dd          d|                     dd          d|                     d          d|                     dd          d|                     d          d|                     d          d|                     d          d|                    d          p|                     d          d|d|d|d|                    dd          d|                    dt          ||z
  d                    d|d	|d|                    d          i}	t          |	|           	 |j	        
                    |	           dS # t          $ r:}t                              d|                     d          |           Y d}~dS d}~ww xY w)zPush a type='async_delegation' event onto the shared completion queue.

    Best-effort: a failure here must not crash the worker, but it WOULD mean a
    silently-lost result, so we log loudly.
    r   process_registryzPAsync delegation %s finished but process_registry import failed; result lost: %srK   Nrz   r{   rP   rq   rw   rx   rL   rM   rN   rO   rD   rF   rG   rH   rI   ro   r   r   r   exit_reasonzHAsync delegation %s: failed to enqueue completion event; result lost: %s)tools.process_registryr  rU   r   r{   rA   rQ   r   rr   completion_queuer   )
r9   rm   ro   r  r   rz   r{   rP   rq   r   s
             r   r   r   K  s   ;;;;;;;   JJ''	
 	
 	

 	 jj##GJJwEJJ//>49;;M::n--<L"O44
 	vzz-44 	

+A2 F F 	VZZ(;<< 	

62&& 	6::i(( 	FJJz** 	

6"" 	G$$;

7(;(; 	& 	7 	  	VZZQ//!" 	FJJl]&BA F F
 
#( 	)* 	+, 	vzz-00- C0 V$$$
)--c22222 
 
 
JJ''	
 	
 	
 	
 	
 	
 	
 	
 	

s,   	 
A/AA#H? ?
J	/I>>JrE   	List[str]c                   t                      t          j                    t          |           }|dk    r| d         n$| dd                    d | D                       z   }|t	          |           ||rt	          |          nd|||||dd|	dd	}t
          5  t          d
 t                                          D                       }||
k    rdd|
 ddcddd           S |t          <   ddd           n# 1 swxY w Y   t          |           t          |
          }dfd}	 |                    t          |                     nh# t          $ r[}t
          5  t                              d           ddd           n# 1 swxY w Y   t                     dd| dcY d}~S d}~ww xY wt                               d||pd           ddS )u+  Dispatch a WHOLE fan-out batch as ONE background unit.

    Unlike ``dispatch_async_delegation`` (which backs a single subagent),
    ``runner`` here runs the entire batch — it builds and joins on every child
    in parallel and returns the combined ``{"results": [...],
    "total_duration_seconds": N}`` dict that the synchronous path would have
    returned. We occupy ONE async slot for the whole batch (the in-batch
    parallelism is bounded separately by ``max_concurrent_children``), so a
    single ``delegate_task`` fan-out never exhausts the async pool by itself.

    When the batch finishes, a SINGLE completion event is pushed onto the
    shared ``process_registry.completion_queue`` carrying the full per-task
    ``results`` list, so the consolidated summaries re-enter the conversation
    as one message once every child is done — the chat is never blocked while
    they run.

    Returns ``{"status": "dispatched", "delegation_id": ...}`` on success or
    ``{"status": "rejected", "error": ...}`` when the async pool is at
    capacity.
    r!   r   z parallel subagents: z; c              3  *   K   | ]}|d d         V  d S )N(   r   )r"   gs     r   r   z2dispatch_async_delegation_batch.<locals>.<genexpr>  s+      G^G^ST#2#G^G^G^G^G^G^r   Nr   T)rK   rD   rE   rF   rG   rH   rI   rL   rN   rO   ro   rP   rq   r   rJ   c              3  L   K   | ]}|                     d           dk    dV   dS r   r@   r   s     r   r   z2dispatch_async_delegation_batch.<locals>.<genexpr>  r   r   r   r   z running). Wait for one to finish (its result will re-enter the chat), or raise delegation.max_concurrent_children in config.yaml to allow more concurrent background units.r   r   r;   c                    i } d}	              pi } |                      d          pg }|rt          d |D                       rd}nd}ns# t          $ rf}t                              d           g t          |          j         d| t          t          j                    z
  d          d} d}Y d }~nd }~ww xY wt          | |           d S # t          | |           w xY w)	Nr{   resultsc              3  D   K   | ]}|                     d           dvV  dS )ro   )rp   successNr@   r   s     r   r   zCdispatch_async_delegation_batch.<locals>._worker.<locals>.<genexpr>  sF       % % x(@@% % % % % %r   rp   z!Async delegation batch %s crashedr   r   )r  r{   total_duration_seconds)
rA   allrU   r   r   rw   r   r   rQ   _finalize_batch)combinedro   child_resultsr   rK   rP   r   s       r   r   z0dispatch_async_delegation_batch.<locals>._worker  s5   #%	=vxx~2H$LL339rM % % %&% % % " " % !$ 	 	 	@-PPP II.77#77*/	m0KQ*O*O H
 FFFFFF	 M8V<<<<<OM8V<<<<s1   AA 
C 
B;AB61C 6B;;C C$z+Failed to schedule async delegation batch: zADispatched async delegation batch %s (%d task(s), session_key=%s)r   r   r   r   )r   rQ   r   joinr   r   r   r   r   r\   r   r   r   rU   r   r`   r   r   )rE   rF   rG   rH   rI   rL   rO   r   rN   r   r   ncombined_goalr9   r   r   r   r   rK   rP   s          `          @@r   dispatch_async_delegation_batchr    s+   D '((MIKKME

A FFa1 ; ; ;diiG^G^X]G^G^G^>^>^ ^  'e&.8DNNND" 4.&$ F" 
 ) ) 
 
((
 
 
 
 
 ((($N:L N N N ) ) ) ) ) ) ) ) #)) ) ) ) ) ) ) ) ) ) ) ) ) ) )  f/00H= = = = = = = =2

3G<<==== 
 
 
 	. 	.LL---	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	."=111 H3HH
 
 	
 	
 	
 	
 	
 	
	
 KKKq+0   #]CCCsZ   ?D *
D  DD2"E 
F:F5&FF5F	F5F	F5/F:5F:r  c                T   t           5  t                              |           }|	 ddd           dS d|d<   t          j                    |d<   d|d<   t	          |          }ddd           n# 1 swxY w Y   	 ddlm} n4# t          $ r'}t          	                    d| |           Y d}~dS d}~ww xY w|                    d	          pt          j                    }|                    d          pt          j                    }i d
dd| d|                    dd          d|                    dd          d|                    d          d|                    dd          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          d|ddd|                    d          pg d|                    d          d|                    d          d	|d|i}	t          |	|           	 |j                            |	           n3# t          $ r&}t          	                    d| |           Y d}~nd}~ww xY wt           5  t                              |           }|||d<   t                       ddd           dS # 1 swxY w Y   dS # t           5  t                              |           }|||d<   t                       ddd           w # 1 swxY w Y   w xY w)zDMark a batch record complete and push ONE combined completion event.Nr   ro   rq   r   r   r   zVAsync delegation batch %s finished but process_registry import failed; result lost: %srP   rw   rx   rK   rL   rM   rN   rO   rD   rE   rF   rG   rH   rI   rJ   Tr  r{   r  zNAsync delegation batch %s: failed to enqueue completion event; result lost: %s)r   r   rA   rQ   r   r  r  rU   r   r{   rr   r  r   r   )
rK   r  ro   r9   r   r  r   rP   rq   r   s
             r   r  r    s    
 $ $m,,>$ $ $ $ $ $ $ $ (x!%~!%~F||$ $ $ $ $ $ $ $ $ $ $ $ $ $ $;;;;;;;   &3	
 	
 	

 	 !$$_55DM##N33Bty{{L" 	|''r:: 	 0 01G L L	
 	\--.ABB 	  ,, 	!!'** 	<##I.. 	L$$Z00 	  (( 	!!'** 	& 	D  	8<<	**0b!" 	g&&#$ 	!(,,/G"H"H%& 	'( 	) C, X&&&&)--c2222 
 
 
3	
 	
 	
 	
 	
 	
 	
 	

  	& 	&\\-00F!#)x #%%%		& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	&] 	& 	&\\-00F!#)x #%%%		& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	&s   A./A..A25A2:B 
B2B--B28I K 
JI>9K >JK 0K

KKL'0LL'LL'"L#L'List[Dict[str, Any]]c                     t           5  d t                                          D             cddd           S # 1 swxY w Y   dS )zSnapshot of async delegations (running + recently completed).

    Safe to call from any thread. Excludes the non-serialisable interrupt_fn.
    c                J    g | ] }d  |                                 D             !S )c                &    i | ]\  }}|d k    ||S )r   r   )r"   kvs      r   rC   z5list_async_delegations.<locals>.<listcomp>.<dictcomp>E  s(    ???da1+>+>Q+>+>+>r   )r   r   s     r   r   z*list_async_delegations.<locals>.<listcomp>D  s?     
 
 
 @?aggii???
 
 
r   N)r   r   r   r   r   r   list_async_delegationsr  >  s    
 
 
 

 
__&&
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   #8<<shutdownreasonc                   d}t           5  d t                                          D             }ddd           n# 1 swxY w Y   |D ]}}|                    d          }t	          |          rW	  |             |dz  }7# t
          $ r9}t                              d|                    d          |           Y d}~ud}~ww xY w~|rt                              d||            |S )	a*  Signal every running async delegation to stop. Returns how many.

    Used on ``/stop`` and gateway shutdown so a dangling background subagent
    can't keep burning tokens with no one listening. The child still emits a
    completion event (status='interrupted') via the normal finalize path.
    r   c                D    g | ]}|                     d           dk    |S r   r@   r   s     r   r   z!interrupt_all.<locals>.<listcomp>S  s3     
 
 
AEE(OOy,H,HA,H,H,Hr   Nr   r!   z&interrupt_all: %s interrupt failed: %srK   z'Interrupted %d async delegation(s) (%s)	r   r   r   rA   callablerU   r   debugr   )r!  counttargetsr   fnr   s         r   interrupt_allr*  J  se    E	 
 

 
((
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
UU>""B<< 	
   <EE/**C       		  N=ufMMMLs%   $:>>.A>>
C/B<<Csession_endc                     sssdS d}t           5   fdt                                          D             }ddd           n# 1 swxY w Y   |D ]}}|                    d          }t	          |          rW	  |             |dz  }7# t
          $ r9}t                              d|                    d          |           Y d}~ud}~ww xY w~|rt                              d||           |S )	u{  Signal running async delegations owned by ONE session to stop.

    A delegation's lifecycle is bound to the session that spawned it: when
    that session ends, its in-flight background subagents must end with it —
    a completed orphan would otherwise sit on the shared completion queue
    with no live owner, either leaking into another chat or burning tokens
    with no one listening (#55578).

    Selectors (any matching field claims the record):
    - ``origin_ui_session_id``: the live TUI tab/window that commissioned it.
    - ``session_key``: the durable routing key captured at dispatch.
    - ``parent_session_id``: the spawning agent's durable session-db id —
      the right selector for gateway chats, whose ``session_key`` (the
      platform conversation key) SURVIVES a ``/new`` reset while the
      session id rotates.

    Returns how many were interrupted.
    r   c                B   g | ]}|                     d           dk    rr(t          |                     d          pd          k    sTr(t          |                     d          pd          k    s*r*t          |                     d          pd          k    |S )ro   r   rN   rM   rL   rO   )rA   r]   )r"   r   rN   rO   rL   s     r   r   z)interrupt_for_session.<locals>.<listcomp>  s     
 
 
uuX)++% ,*-aee4J.K.K.Qr*R*RVj*j*j +k$'m(<(<(B$C$C{$R$R% %S*-aee4G.H.H.NB*O*OSd*d*d 
 +e*d*dr   Nr   r!   z.interrupt_for_session: %s interrupt failed: %srK   z:Interrupted %d async delegation(s) for ending session (%s)r$  )	rL   rN   rO   r!  r'  r(  r   r)  r   s	   ```      r   interrupt_for_sessionr.  f  s   0  3 <M qE	 	
 	

 
 
 
 
 
((
 
 
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
  
 
UU>""B<< 	
   DEE/**C       		  
H6	
 	
 	
 Ls)   (A		AA=B
C/CCc                    t           5  t          t                              d           dadaddd           n# 1 swxY w Y   t          5  t
                                           ddd           dS # 1 swxY w Y   dS )z6Test-only: clear all state and tear down the executor.NF)waitr   )r   r   r   r   r   r   clearr   r   r   _reset_for_testsr2    s    
 " " E***	 !	" " " " " " " " " " " " " " "
 
                   s   ';??A44A8;A8)r   r   )r9   r:   r   r;   )rK   r]   r   r;   r   )rl   r:   rm   r:   r   r;   )r   r   )rK   r]   r   r~   )rK   r]   r   r]   r   r~   )r   r:   r   r]   r   r   )r   r:   r   r]   r   r;   )rK   r]   r   r   )r   r   r   r   )r   r]   )rD   r]   rF   r   rG   r   rH   r]   rI   r   rL   r]   rO   r   r   r   rN   r]   r   r   r   r   r   r:   )rK   r]   rm   r:   ro   r]   r   r;   )r9   r:   rm   r:   ro   r]   r   r;   )rE   r  rF   r   rG   r   rH   r]   rI   r   rL   r]   rO   r   r   r   rN   r]   r   r   r   r   r   r:   )rK   r]   r  r:   ro   r]   r   r;   )r   r  )r   )r!  r]   r   r   )rM   rM   rM   r+  )
rL   r]   rN   r]   rO   r]   r!  r]   r   r   )E__doc__
__future__r   rW   loggingr0   	threadingrQ   r   concurrent.futuresr   typingr   r   r   r   r	   hermes_constantsr   tools.daemon_poolr   tools.thread_contextr   	getLoggerr   r   r   r   __annotations__Lockr   r   r   r   _DEFAULT_MAX_ASYNC_CHILDRENre   rb   rf   rV   r   r8   r\   r`   rY   rr   rt   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r*  r.  r2  r   r   r   <module>r@     sD  ! ! !F # " " " " "          1 1 1 1 1 1 6 6 6 6 6 6 6 6 6 6 6 6 6 6 , , , , , , 6 6 6 6 6 6 < < < < < <		8	$	$
 5  +/	 . . . .!!     	   ') ( ( ( (  -  9>* * *% % % %P   8^ ^ ^ ^
$ $ $ $N	
 	
 	
 	

 
 
 
, , , ,^   6	! 	! 	! 	!! ! ! !(T T T T
! 
! 
! 
!! ! ! !T T T T
S S S S
   "   &a a a a+ + + +       4 (, "159|D |D |D |D |D |D~" " " ",7
 7
 7
 7
D (, "159vD vD vD vD vD vDr=& =& =& =&@	
 	
 	
 	
    :  "	5 5 5 5 5p	 	 	 	 	 	r   