Rebuild: rules:signatures, and voyager 1's 29 learned garbage signatures
rules:signatures publishes a rule set version whose dropped signatures come from a file of tool<TAB>group lines (--add to add to the current ones). database/data/voyager1_signatures.tsv: voyager 1's dropped ones (>=100 probes, <=0.5% named), with their probe counts; the two builds name the posting tools the same way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fd0f2bde75
commit
57dfa3d749
2 files changed
+82
No files matched your search
@@ -0,0 +1,52 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { signatureKey } from '#lib/rules/index'
|
||||
|
||||
/**
|
||||
* Sets the dropped upload signatures (REBUILD.md, rules and recovery) from a
|
||||
* file of "tool<TAB>group" lines, e.g. voyager 1's learned ones:
|
||||
*
|
||||
* psql … -At -F $'\t' -c "SELECT tool, group_name FROM upload_signatures WHERE dropped"
|
||||
*
|
||||
* Publishes a new rule set version (ranges record the version they ran with).
|
||||
*/
|
||||
export default class RulesSignatures extends BaseCommand {
|
||||
static commandName = 'rules:signatures'
|
||||
static description = 'Publishes a rule set with the dropped upload signatures from a file'
|
||||
static options: CommandOptions = { startApp: true }
|
||||
|
||||
@args.string({ description: 'File of "tool<TAB>group" lines (more columns and # lines are ignored), e.g. database/data/voyager1_signatures.tsv' })
|
||||
declare file: string
|
||||
|
||||
@flags.string({ description: 'Note stored with the rule set version' })
|
||||
declare note?: string
|
||||
|
||||
@flags.boolean({ description: 'Add to the current signatures instead of replacing them' })
|
||||
declare add: boolean
|
||||
|
||||
async run() {
|
||||
const lines = (await readFile(this.file, 'utf8')).split('\n').map((l) => l.trim()).filter((l) => l && !l.startsWith('#'))
|
||||
const signatures = new Set<string>()
|
||||
for (const line of lines) {
|
||||
const [tool, group] = line.split('\t')
|
||||
if (!tool || !group) throw new Error(`Not "tool<TAB>group": ${line}`)
|
||||
signatures.add(signatureKey(tool, group))
|
||||
}
|
||||
const { version: _v, priorities: _p, ...current } = await RuleSets.current()
|
||||
const before = new Set(current.droppedSignatures)
|
||||
const next = this.add ? new Set([...before, ...signatures]) : signatures
|
||||
if (next.size === before.size && [...next].every((s) => before.has(s))) {
|
||||
this.logger.info(`Unchanged: ${before.size} dropped signatures already.`)
|
||||
return
|
||||
}
|
||||
const version = await RuleSets.publish(
|
||||
{ ...current, droppedSignatures: [...next].sort() },
|
||||
this.note ?? `${next.size} dropped signatures from ${this.file}`
|
||||
)
|
||||
const added = [...next].filter((s) => !before.has(s)).length
|
||||
const removed = [...before].filter((s) => !next.has(s)).length
|
||||
this.logger.info(`Rule set ${version}: ${next.size} dropped signatures (${added} added, ${removed} removed).`)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user