Skip to content

Commit

Permalink
update doonce logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@StardustDL
StardustDL committed Feb 19, 2024
1 parent 36bb702 commit c3d5cf8
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 88 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ jobs:
!./cache/*/wheels
- name: Remove logs
run: |
rm ./dist/**/*.log
rm ./dist/*/*/*.log
- name: Upload pages artifact
uses: actions/upload-pages-artifact@v3
with:
Expand Down
33 changes: 20 additions & 13 deletions index/processor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,21 +98,24 @@ def workerVersion(self):
def hasDone(self, type: str, id: str):
return f"{type}:{id}" in self.db.data

@contextmanager
def doOnce(self, type: str, id: str):
with self.db.do(f"{type}:{id}", self.workerVersion) as res:
if res:
assert res.state == ProcessState.SUCCESS, "Not success"
yield res
if self.hasDone(type, id):
return None
@contextmanager
def wrapper():
with self.db.do(f"{type}:{id}", self.workerVersion):
yield
return wrapper

def version(self, release: Release):
env.logger.info(f"Process release {release}")
dis = self.cacheDist.preprocess(release)
api = self.cacheDist.extract(release)
wheelDir = self.cacheDist.projectDir(release.project) / "wheels"
utils.ensureDirectory(wheelDir)
with self.doOnce(JOB_PREPROCESS, str(release)) as _:
if _ is None:
wrapper = self.doOnce(JOB_PREPROCESS, str(release))
if wrapper:
with wrapper():
env.logger.info(f"Preprocess release {release}")
result = self.worker.preprocess(
[
Expand All @@ -125,8 +128,9 @@ def version(self, release: Release):
)
result.save(dis)
result.ensure().save(self.dist.preprocess(release))
with self.doOnce(JOB_EXTRACT, str(release)) as _:
if _ is None:
wrapper = self.doOnce(JOB_EXTRACT, str(release))
if wrapper:
with wrapper():
env.logger.info(f"Extract release {release}")
result = self.worker.extract([str(self.worker.resolvePath(dis)), "-"])
result.save(api)
Expand All @@ -139,8 +143,10 @@ def pair(self, pair: ReleasePair):
new = self.cacheDist.extract(pair.new)
cha = self.cacheDist.diff(pair)
rep = self.cacheDist.report(pair)
with self.doOnce(JOB_DIFF, str(pair)) as _:
if _ is None:

wrapper = self.doOnce(JOB_DIFF, str(pair))
if wrapper:
with wrapper():
env.logger.info(f"Diff releas pair {pair}")
result = self.worker.diff(
[
Expand All @@ -151,8 +157,9 @@ def pair(self, pair: ReleasePair):
)
result.save(cha)
result.ensure().save(self.dist.diff(pair))
with self.doOnce(JOB_REPORT, str(pair)) as _:
if _ is None:
wrapper = self.doOnce(JOB_REPORT, str(pair))
if wrapper:
with wrapper():
env.logger.info(f"Report releas pair {pair}")
result = self.worker.report([str(self.worker.resolvePath(cha)), "-"])
result.save(rep)
Expand Down
147 changes: 73 additions & 74 deletions index/std.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,86 +50,83 @@ def __init__(self, db: ProcessDB, dist: DistPathBuilder) -> None:

@override
def version(self, release):
need = not self.hasDone(JOB_EXTRACT, str(release))
if not need:
wrapper = self.doOnce(JOB_EXTRACT, str(release))
if not wrapper:
return

dis = self.cacheDist.preprocess(release)
api = self.cacheDist.extract(release)

with self.envBuilder.use(release.version, logger=env.logger) as e:
with e as r:
with self.doOnce(JOB_EXTRACT, str(release)) as _:
if _ is None:
pathRes = r.runPythonText(
'-c "import pathlib; print(pathlib.__file__)"', check=True
)
rootPath = Path(pathRes.stdout.strip()).parent
modules = list(getTopModules(rootPath))

result = self.worker.preprocess(
[
"-s",
str(rootPath),
"-p",
str(release),
"-P",
release.version,
*sum([["-m", m] for m in modules], start=[]),
"-",
]
)
result.save(dis)
result.ensure().save(self.dist.preprocess(release))

removeMain(rootPath)

totalResult: ApiDescription | None = None
totalLog = ""

with utils.elapsedTimer() as timer:
for module in modules:
env.logger.info(f"Process stdlib: {module}")
result = self.worker.preprocess(
[
"-s",
str(rootPath),
"-p",
str(release),
"-P",
release.version,
"-m",
module,
"-",
],
check=True,
)
result = self.worker.extract(
["-", "-", "-e", e.name], input=result.out
)
totalLog += result.log

if result.data is None:
continue

if totalResult is None:
totalResult = result.data
else:
for entry in result.data.entries.values():
if entry.id not in totalResult.entries:
totalResult.addEntry(entry)
if totalResult is None:
finalResult = AexPyResult(
code=1, log="Failed to dump", out=""
with wrapper():
pathRes = r.runPythonText(
'-c "import pathlib; print(pathlib.__file__)"', check=True
)
rootPath = Path(pathRes.stdout.strip()).parent
modules = list(getTopModules(rootPath))

result = self.worker.preprocess(
[
"-s",
str(rootPath),
"-p",
str(release),
"-P",
release.version,
*sum([["-m", m] for m in modules], start=[]),
"-",
]
)
result.save(dis)
result.ensure().save(self.dist.preprocess(release))

removeMain(rootPath)

totalResult: ApiDescription | None = None
totalLog = ""

with utils.elapsedTimer() as timer:
for module in modules:
env.logger.info(f"Process stdlib: {module}")
result = self.worker.preprocess(
[
"-s",
str(rootPath),
"-p",
str(release),
"-P",
release.version,
"-m",
module,
"-",
],
check=True,
)
else:
totalResult.duration = timer()
finalResult = AexPyResult(
out=totalResult.model_dump_json(), log=totalLog, code=0
result = self.worker.extract(
["-", "-", "-e", e.name], input=result.out
)
totalLog += result.log

if result.data is None:
continue

if totalResult is None:
totalResult = result.data
else:
for entry in result.data.entries.values():
if entry.id not in totalResult.entries:
totalResult.addEntry(entry)
if totalResult is None:
finalResult = AexPyResult(code=1, log="Failed to dump", out="")
else:
totalResult.duration = timer()
finalResult = AexPyResult(
out=totalResult.model_dump_json(), log=totalLog, code=0
)

finalResult.save(api)
finalResult.ensure().save(self.dist.extract(release))
finalResult.save(api)
finalResult.ensure().save(self.dist.extract(release))

@override
def pair(self, pair):
Expand All @@ -143,13 +140,15 @@ def pair(self, pair):
newA = self.cacheDist.extract(pair.new)
cha = self.cacheDist.diff(pair)
rep = self.cacheDist.report(pair)
with self.doOnce(JOB_DIFF, str(pair)) as _:
if _ is None:
wrapper = self.doOnce(JOB_DIFF, str(pair))
if wrapper:
with wrapper():
result = self.worker.diff([str(oldA), str(newA), "-"])
result.save(cha)
result.ensure().save(self.dist.diff(pair))
with self.doOnce(JOB_REPORT, str(pair)) as _:
if _ is None:
wrapper = self.doOnce(JOB_REPORT, str(pair))
if wrapper:
with wrapper():
result = self.worker.report([str(cha), "-"])
result.save(rep)
result.ensure().save(self.dist.report(pair))
Expand Down

0 comments on commit c3d5cf8

Please sign in to comment.