morgan/Grinch-AP
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WebHost: fix gen timeout/exception resource handling (#5540)

Browse Source 
* WebHost: reset Generator proc title on error

* WebHost: fix shutting down autogen

This is still not perfect but solves some of the issues.

* WebHost: properly propagate JOB_TIME

* WebHost: handle autogen shutdown
This commit is contained in:
black-sliver
2025-10-20 07:16:29 +00:00
committed by GitHub
parent 11d18db452
commit 914a534a3b
4 changed files with 93 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
WebHostLib/autolauncher.py
View File

@@ -36,25 +36,39 @@ def handle_generation_failure(result: BaseException):
logging.exception(e)
def _mp_gen_game(gen_options: dict, meta: dict[str, Any] | None = None, owner=None, sid=None) -> PrimaryKey | None:
def _mp_gen_game(
gen_options: dict,
meta: dict[str, Any] | None = None,
owner=None,
sid=None,
timeout: int|None = None,
) -> PrimaryKey | None:
from setproctitle import setproctitle
setproctitle(f"Generator ({sid})")
res = gen_game(gen_options, meta=meta, owner=owner, sid=sid)
setproctitle(f"Generator (idle)")
return res
try:
return gen_game(gen_options, meta=meta, owner=owner, sid=sid, timeout=timeout)
finally:
setproctitle(f"Generator (idle)")
def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation):
def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation, timeout: int|None) -> None:
try:
meta = json.loads(generation.meta)
options = restricted_loads(generation.options)
logging.info(f"Generating {generation.id} for {len(options)} players")
pool.apply_async(_mp_gen_game, (options,),
{"meta": meta,
"sid": generation.id,
"owner": generation.owner},
handle_generation_success, handle_generation_failure)
pool.apply_async(
_mp_gen_game,
(options,),
{
"meta": meta,
"sid": generation.id,
"owner": generation.owner,
"timeout": timeout,
},
handle_generation_success,
handle_generation_failure,
)
except Exception as e:
generation.state = STATE_ERROR
commit()
@@ -135,6 +149,7 @@ def autogen(config: dict):
with multiprocessing.Pool(config["GENERATORS"], initializer=init_generator,
initargs=(config,), maxtasksperchild=10) as generator_pool:
job_time = config["JOB_TIME"]
with db_session:
to_start = select(generation for generation in Generation if generation.state == STATE_STARTED)
@@ -145,7 +160,7 @@ def autogen(config: dict):
if sid:
generation.delete()
else:
launch_generator(generator_pool, generation)
launch_generator(generator_pool, generation, timeout=job_time)
commit()
select(generation for generation in Generation if generation.state == STATE_ERROR).delete()
@@ -157,7 +172,7 @@ def autogen(config: dict):
generation for generation in Generation
if generation.state == STATE_QUEUED).for_update()
for generation in to_start:
launch_generator(generator_pool, generation)
launch_generator(generator_pool, generation, timeout=job_time)
except AlreadyRunningException:
logging.info("Autogen reports as already running, not starting another.")