From 7e485db88e8179a29172bb7675d93e51699c506b Mon Sep 17 00:00:00 2001 From: "tho.nguyen" <91511523+haki203@users.noreply.github.com> Date: Fri, 22 May 2026 22:44:57 +0700 Subject: [PATCH] Add peer review template rubric guard --- peer-review-template-rubric-guard/README.md | 33 +++ peer-review-template-rubric-guard/demo.js | 147 ++++++++++ peer-review-template-rubric-guard/index.js | 262 ++++++++++++++++++ .../reports/demo.mp4 | Bin 0 -> 6609 bytes .../reports/rubric-guard-packet.json | 224 +++++++++++++++ .../reports/rubric-guard-review.md | 59 ++++ .../reports/summary.svg | 23 ++ .../requirements-map.md | 18 ++ .../sample-data.js | 68 +++++ peer-review-template-rubric-guard/test.js | 110 ++++++++ 10 files changed, 944 insertions(+) create mode 100644 peer-review-template-rubric-guard/README.md create mode 100644 peer-review-template-rubric-guard/demo.js create mode 100644 peer-review-template-rubric-guard/index.js create mode 100644 peer-review-template-rubric-guard/reports/demo.mp4 create mode 100644 peer-review-template-rubric-guard/reports/rubric-guard-packet.json create mode 100644 peer-review-template-rubric-guard/reports/rubric-guard-review.md create mode 100644 peer-review-template-rubric-guard/reports/summary.svg create mode 100644 peer-review-template-rubric-guard/requirements-map.md create mode 100644 peer-review-template-rubric-guard/sample-data.js create mode 100644 peer-review-template-rubric-guard/test.js diff --git a/peer-review-template-rubric-guard/README.md b/peer-review-template-rubric-guard/README.md new file mode 100644 index 00000000..3833d63d --- /dev/null +++ b/peer-review-template-rubric-guard/README.md @@ -0,0 +1,33 @@ +# Peer Review Template Rubric Guard + +Self-contained reviewer module for issue #15, Community & User Reputation System. + +The guard validates discipline-specific structured peer-review templates before their reviews can affect project timelines, reviewer profiles, citation pages, or reputation deltas. + +## What It Does + +- Holds templates that miss required rubric dimensions: clarity, rigor, novelty, and reproducibility. +- Blocks score scales outside a comparable 0..10 range. +- Requires rubric evidence anchors for reviewed documents, datasets, code, and notebooks. +- Routes anonymous or double-blind privacy gaps to curator review. +- Requires reviewer declarations for conflicts, method expertise, and data access. +- Checks profile/citation publication packets for rubric versioning and anonymous identity redaction. +- Generates deterministic JSON, Markdown, SVG, and MP4 artifacts from synthetic fixtures only. + +## Files + +- `index.js` - dependency-free rubric evaluator and Markdown packet builder. +- `sample-data.js` - synthetic templates for held, curator-review, and publish decisions. +- `test.js` - Node tests for blocking, review-only, and publication-ready templates. +- `demo.js` - report generator and optional MP4 artifact writer. +- `requirements-map.md` - issue requirement mapping. +- `reports/` - generated reviewer artifacts. + +## Run + +```bash +node --test peer-review-template-rubric-guard/test.js +FFMPEG_PATH=/path/to/ffmpeg node peer-review-template-rubric-guard/demo.js +``` + +No live reviewers, profiles, projects, identities, credentials, external services, or network calls are used. diff --git a/peer-review-template-rubric-guard/demo.js b/peer-review-template-rubric-guard/demo.js new file mode 100644 index 00000000..49262f43 --- /dev/null +++ b/peer-review-template-rubric-guard/demo.js @@ -0,0 +1,147 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); + +const { + evaluatePeerReviewTemplateRubric, + buildRubricGuardPacket, +} = require('./index'); +const {scenarios} = require('./sample-data'); + +const reportsDir = path.join(__dirname, 'reports'); +const framesDir = path.join(reportsDir, 'frames'); +fs.mkdirSync(reportsDir, {recursive: true}); + +const evaluations = scenarios.map((scenario) => ({ + scenario: scenario.name, + ...evaluatePeerReviewTemplateRubric(scenario), +})); + +const decisionCounts = evaluations.reduce((counts, item) => { + counts[item.decision] = (counts[item.decision] || 0) + 1; + return counts; +}, {}); +const totals = evaluations.reduce( + (sum, item) => { + sum.findings += item.summary.findingCount; + sum.blocking += item.summary.blockingFindingCount; + sum.review += item.summary.reviewFindingCount; + sum.actions += item.curatorActions.length; + return sum; + }, + {findings: 0, blocking: 0, review: 0, actions: 0} +); + +const packetJson = JSON.stringify(evaluations, null, 2); +const reviewerPacket = evaluations.map(buildRubricGuardPacket).join('\n---\n'); +const svg = ` + + Peer Review Template Rubric Guard + Validates structured peer-review templates before profile, timeline, citation, or reputation updates + + + Held + ${decisionCounts['hold-template'] || 0} + + + + Curator Review + ${decisionCounts['curator-review'] || 0} + + + + Publish + ${decisionCounts['publish-template'] || 0} + + Findings: ${totals.findings} | Blocking: ${totals.blocking} | Review: ${totals.review} | Curator actions: ${totals.actions} + Checks: required rubric dimensions, score scales, evidence anchors, review-mode privacy, declarations, profile packets + Synthetic templates only. No live reviewers, profiles, projects, identities, credentials, or external services. + +`; + +fs.writeFileSync(path.join(reportsDir, 'rubric-guard-packet.json'), `${packetJson}\n`); +fs.writeFileSync(path.join(reportsDir, 'rubric-guard-review.md'), reviewerPacket); +fs.writeFileSync(path.join(reportsDir, 'summary.svg'), svg); + +function fillRect(buffer, width, x0, y0, rectWidth, rectHeight, color) { + const [r, g, b] = color; + const height = Math.floor(buffer.length / (width * 3)); + const x1 = Math.min(width, x0 + rectWidth); + const y1 = Math.min(height, y0 + rectHeight); + for (let y = Math.max(0, y0); y < y1; y += 1) { + for (let x = Math.max(0, x0); x < x1; x += 1) { + const offset = (y * width + x) * 3; + buffer[offset] = r; + buffer[offset + 1] = g; + buffer[offset + 2] = b; + } + } +} + +function writePpmFrame(filePath, frameIndex, frameCount) { + const width = 640; + const height = 360; + const buffer = Buffer.alloc(width * height * 3); + for (let i = 0; i < width * height; i += 1) { + buffer[i * 3] = 23; + buffer[i * 3 + 1] = 32; + buffer[i * 3 + 2] = 42; + } + + const progress = frameIndex / Math.max(1, frameCount - 1); + fillRect(buffer, width, 42, 42, 556, 42, [248, 250, 252]); + fillRect(buffer, width, 42, 112, 150, 118, [127, 29, 29]); + fillRect(buffer, width, 245, 112, 150, 118, [133, 77, 14]); + fillRect(buffer, width, 448, 112, 150, 118, [22, 101, 52]); + fillRect(buffer, width, 78, 260, 112, 30, [248, 113, 113]); + fillRect(buffer, width, 264, 260, 112, 30, [251, 191, 36]); + fillRect(buffer, width, 450, 260, 112, 30, [74, 222, 128]); + fillRect(buffer, width, 42, 318, Math.round(556 * progress), 18, [191, 219, 254]); + fillRect(buffer, width, 42 + Math.round(508 * progress), 309, 48, 36, [241, 245, 249]); + + const header = Buffer.from(`P6\n${width} ${height}\n255\n`); + fs.writeFileSync(filePath, Buffer.concat([header, buffer])); +} + +function createDemoVideo() { + const ffmpegPath = process.env.FFMPEG_PATH; + if (!ffmpegPath) { + console.log('FFMPEG_PATH not set; skipped MP4 generation.'); + return; + } + + fs.rmSync(framesDir, {recursive: true, force: true}); + fs.mkdirSync(framesDir, {recursive: true}); + const frameCount = 72; + for (let index = 0; index < frameCount; index += 1) { + writePpmFrame(path.join(framesDir, `frame-${String(index).padStart(3, '0')}.ppm`), index, frameCount); + } + + const output = path.join(reportsDir, 'demo.mp4'); + const result = spawnSync(ffmpegPath, [ + '-y', + '-framerate', + '24', + '-i', + path.join(framesDir, 'frame-%03d.ppm'), + '-c:v', + 'libx264', + '-pix_fmt', + 'yuv420p', + output, + ], {encoding: 'utf8'}); + + fs.rmSync(framesDir, {recursive: true, force: true}); + if (result.status !== 0) { + throw new Error(result.stderr || 'ffmpeg failed'); + } +} + +createDemoVideo(); + +console.log(JSON.stringify({ + scenarios: evaluations.length, + reportsDir, + decisions: decisionCounts, + totals, +}, null, 2)); diff --git a/peer-review-template-rubric-guard/index.js b/peer-review-template-rubric-guard/index.js new file mode 100644 index 00000000..8695db72 --- /dev/null +++ b/peer-review-template-rubric-guard/index.js @@ -0,0 +1,262 @@ +const REQUIRED_DIMENSIONS = ['clarity', 'rigor', 'novelty', 'reproducibility']; +const REQUIRED_DECLARATIONS = ['conflictOfInterest', 'methodExpertise', 'dataAccess']; + +function list(value) { + return Array.isArray(value) ? value : []; +} + +function clean(value) { + return String(value || '').trim(); +} + +function normalize(value) { + return clean(value).toLowerCase(); +} + +function finding(type, severity, subject, message) { + return {type, severity, subject, message}; +} + +function action(type, target, reason) { + return {type, target, reason}; +} + +function dimensionNames(input) { + return new Set(list(input.dimensions).map((dimension) => normalize(dimension.name))); +} + +function validateRequiredDimensions(input, findings, curatorActions) { + const names = dimensionNames(input); + for (const required of REQUIRED_DIMENSIONS) { + if (!names.has(required)) { + findings.push( + finding( + 'missing-required-rubric-dimension', + 'block', + required, + `Template is missing the required ${required} rubric dimension.` + ) + ); + curatorActions.push( + action( + 'add-rubric-dimension', + required, + `Add a ${required} scoring dimension before this peer-review template can affect reputation.` + ) + ); + } + } +} + +function validateScoreScales(input, findings, curatorActions) { + for (const dimension of list(input.dimensions)) { + const scale = dimension.scoreScale || {}; + const min = Number(scale.min); + const max = Number(scale.max); + if (!Number.isFinite(min) || !Number.isFinite(max) || min < 0 || max > 10 || min >= max) { + findings.push( + finding( + 'score-scale-out-of-bounds', + 'block', + dimension.name || 'unknown dimension', + `${dimension.name || 'Rubric dimension'} uses score scale ${scale.min}..${scale.max}, outside the supported 0..10 increasing range.` + ) + ); + curatorActions.push( + action( + 'normalize-score-scale', + dimension.name || 'unknown dimension', + 'Use an increasing numeric score scale within 0..10 so reputation deltas are comparable.' + ) + ); + } + } +} + +function validateEvidenceAnchors(input, findings, curatorActions) { + const coveredArtifacts = new Set(); + for (const dimension of list(input.dimensions)) { + for (const anchor of list(dimension.evidenceAnchors)) { + coveredArtifacts.add(normalize(anchor)); + } + } + + for (const artifactType of list(input.reviewedArtifactTypes)) { + if (!coveredArtifacts.has(normalize(artifactType))) { + findings.push( + finding( + 'evidence-anchor-missing', + 'block', + artifactType, + `No rubric dimension requires evidence anchored to ${artifactType}.` + ) + ); + curatorActions.push( + action( + 'add-evidence-anchor', + artifactType, + `Require at least one scoring dimension to cite ${artifactType} evidence before profile or timeline credit is computed.` + ) + ); + } + } +} + +function validateReviewMode(input, findings, curatorActions) { + const visibility = normalize(input.visibilityMode); + const mode = input.reviewMode || {}; + const requiresBlindSafety = visibility.includes('blind') || visibility.includes('anonymous'); + const safeLabel = !['real-name', 'public-name'].includes(normalize(mode.anonymousLabelPolicy)); + if (requiresBlindSafety && (!mode.authorIdentityHidden || !mode.reviewerIdentityHidden || !safeLabel)) { + findings.push( + finding( + 'anonymous-mode-privacy-gap', + 'review', + input.visibilityMode || 'review mode', + 'Anonymous or blind review mode does not hide both author and reviewer identities with a safe label policy.' + ) + ); + curatorActions.push( + action( + 'repair-review-mode-privacy', + input.templateId || 'template', + 'Hide author and reviewer identities and replace real-name labels before publishing review receipts.' + ) + ); + } +} + +function validateReviewerDeclarations(input, findings, curatorActions) { + const declarations = input.reviewerDeclarations || {}; + for (const declaration of REQUIRED_DECLARATIONS) { + if (declarations[declaration] !== true) { + findings.push( + finding( + 'reviewer-declaration-incomplete', + 'review', + declaration, + `Reviewer declaration ${declaration} must be completed before reputation deltas are trusted.` + ) + ); + curatorActions.push( + action( + 'complete-reviewer-declaration', + declaration, + 'Require reviewer attestations before template output affects profiles or leaderboards.' + ) + ); + } + } +} + +function validateProfilePacket(input, findings, curatorActions) { + const targets = new Set(list(input.publicationTargets).map(normalize)); + const packet = input.profilePacket || {}; + const needsProfileSafety = targets.has('reviewer-profile') || targets.has('citation-page'); + if (needsProfileSafety && (!packet.includesRubricVersion || !packet.redactsAnonymousIdentities)) { + findings.push( + finding( + 'profile-publication-packet-incomplete', + 'review', + input.templateId || 'template', + 'Profile or citation-page publication packet must include rubric versioning and anonymous identity redaction.' + ) + ); + curatorActions.push( + action( + 'complete-profile-publication-packet', + input.templateId || 'template', + 'Include rubric version metadata and anonymous identity redaction before profile or citation publication.' + ) + ); + } +} + +function summarize(findings) { + const blockingFindingCount = findings.filter((item) => item.severity === 'block').length; + const reviewFindingCount = findings.filter((item) => item.severity !== 'block').length; + return { + findingCount: findings.length, + blockingFindingCount, + reviewFindingCount, + }; +} + +function evaluatePeerReviewTemplateRubric(input = {}) { + const findings = []; + const curatorActions = []; + + validateRequiredDimensions(input, findings, curatorActions); + validateScoreScales(input, findings, curatorActions); + validateEvidenceAnchors(input, findings, curatorActions); + validateReviewMode(input, findings, curatorActions); + validateReviewerDeclarations(input, findings, curatorActions); + validateProfilePacket(input, findings, curatorActions); + + const summary = summarize(findings); + const decision = summary.blockingFindingCount > 0 + ? 'hold-template' + : summary.reviewFindingCount > 0 + ? 'curator-review' + : 'publish-template'; + const profileUnsafeFindingTypes = new Set(['anonymous-mode-privacy-gap', 'profile-publication-packet-incomplete']); + const profileSafe = !findings.some((item) => profileUnsafeFindingTypes.has(item.type)); + + return { + generatedAt: input.generatedAt || new Date(0).toISOString(), + templateId: input.templateId || 'unknown-template', + discipline: input.discipline || 'unknown-discipline', + decision, + summary, + findings, + curatorActions, + templatePacket: { + templateId: input.templateId || 'unknown-template', + discipline: input.discipline || 'unknown-discipline', + decision, + profileSafe, + rubricScore: Math.max(0, 100 - summary.blockingFindingCount * 15 - summary.reviewFindingCount * 8), + requiredDimensions: REQUIRED_DIMENSIONS, + reviewedArtifactTypes: list(input.reviewedArtifactTypes), + publicationTargets: list(input.publicationTargets), + }, + }; +} + +function buildRubricGuardPacket(result) { + const lines = [ + `# Peer Review Template Rubric Guard: ${result.templateId}`, + '', + `Discipline: ${result.discipline}`, + `Decision: ${result.decision}`, + `Generated: ${result.generatedAt}`, + `Rubric score: ${result.templatePacket.rubricScore}`, + `Profile safe: ${result.templatePacket.profileSafe}`, + '', + '## Findings', + ]; + + if (result.findings.length === 0) { + lines.push('- None'); + } else { + for (const item of result.findings) { + lines.push(`- ${item.type}: ${item.subject} - ${item.message}`); + } + } + + lines.push('', '## Curator Actions'); + if (result.curatorActions.length === 0) { + lines.push('- No curator action required'); + } else { + for (const item of result.curatorActions) { + lines.push(`- ${item.type}: ${item.target} - ${item.reason}`); + } + } + + return `${lines.join('\n')}\n`; +} + +module.exports = { + evaluatePeerReviewTemplateRubric, + buildRubricGuardPacket, +}; diff --git a/peer-review-template-rubric-guard/reports/demo.mp4 b/peer-review-template-rubric-guard/reports/demo.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8d699586726e31b73deab140867ea82c0c9c5c5 GIT binary patch literal 6609 zcmcIp2{@Ep`+vqxDU?)}MwZGtGnOok45_@WDne?EnZYn)#>^NLZwP6T`cw+hszrrL zt0*M2mlSzJsoqMosk~n1yU&bineV;6@4Eii^*>klob#Ofci-n;&biOy3Bxdas4$+# z5^y;frhuUyh(r_7i5#9i5yLRGP(G81VPlFo44M!yB`V5dadDg1>&q`&=APSW^%&n` z($gD9atsL)ct;B$?03rCnb69aq1{G-}f&`UMiwa|siH>-BIG@X*Q9+c57xI~G zHcJ2_r#L4DT?hkuG>1$86KIUp+$biQM6@Cj@u4(`#J@oX+Pk`@k6)bEpc0h>h!V`LKqd=y`@2%W{D36UYJC?S*2rU3}%L)bBV zT0E7`!w&ZkqqyNmdII+@N7#dAU+u*fZVAd~ED2>58^44Gi#2tyuHEa_=+ zWJhP15D1w(vOS)~gDk<0fFy!0S~P3`luj%O!<3wv8L611c)VX&NASviLw)mmNY~7FlUsYo zEjqUI_b>*gg()}fYPjQFaNqrAMhT;?g^l&w9;xoqYX1=!HtNd$OkeSF@QulIOM9y^r%CXsmWX^K`{Y7TuDC1MfH zF+Ke)_@JOC3B0h4bFu-fXS*Zc9~tmD_l|?ay~4mFTK{FyOL+gn48^{#H^J_Tk_Inte)!tkmlP0RtAw_FGZAfff_Gz_yn^MyKC+A}_grO-{ zuDl#)Qq>&(#>;fwN^8B$6E-EYVzav2jcZ%{w1Rdp8ly8dY;SQ&)YkS{D_XIpTBmD! z@(9(WS@|xSPc|RB9>E%#*q*63i;FiKJ@1wNo`6APH}VcnEambRvrLm}LB=M#DSwaW zG&QuV2I-QPmMsk2y1X{ubbC|5svu21H|2))m=vGAo0E!^_;*SY_3l^{?L9QHHqr7( z)A=_4G360S>;3zZ%G~EE`>JpBc>2z-$;)8vRdHSIxV|9Ln#1E=#M(t~CbH)%V|osU zn0_7`4es~bjQd36k5ABD)|_DVHHhMnclfU_zImJ4ZX^g^*>yd1Qht@K$i2Uo9sVM* z?v}7MLE~U_`UCS{HJ~R7JGw(>jMv_UetF}$ZB<}`xraJB3UolDuFxY{fi66tFTj% z&tuva;CRf%C&s!P{cd*BVOHxAkC2Kn`x1*5;ycdXC@#`2o^5e>N`3Ex%;R6ny088% zSVjEsv1A7&U5l~rZNjtLH}Xr4lGntz1al1dyo>~<(JP%cQ@8v-6fMh?DCPaf%UqwX zyiP7lr&^bKZ~YQG^KtXTrhR_|;w2y>*&z z71R2Cr-@X|RJ(hP6#R7#6iu+JG0zjM^=i;3Pw`pFSigX@ZJPgvT5rqp!b{9EE7$zy z8N#y)n3TKkd9(kl9Wke@IXL6_dqz&>t{ZYqoafdU@bTT`lfh!`&)+s01UznkI<<&( z=ZI z=BoQ?08hxfvGlNS9Nj$1MeT0Kh=R0$#;68G;WZt01-3rW9L!lt$@QFJoINddig@sY zq-KqZsV%m}lnDdBUF-95;$eqA-z#=_>sNovd&wc?2A{BdmCX-qmKXiqV-~A=b=d`O z-)AOWpnCmg?hBF2GpFEbS;nAt0AET#kla~UO<;oUjf ze5r!27{vi9qq6u~XGcG0kIHX{rZ%ij$r0;TY}F9qhHQOBDf7b?PPu0I#@IXmiq+Kg zI*%`r`^1&!M%l)Cr#w8(N^O}c8X0Lp1HDrzxoPrxZ`-TSCyyXZ=(Igf9l7j?R;HGY zcpG4?DY-@R*hlR(YC0!d4{kmEB5Ci;na5M;^&hhrzNgMc9yw5Q>m>rp+B2Z`51W0u zy+$Xk`0P=uib{WtL%4xT*AMEG9z>UL67v_bbPbY05RWT(B@0DmhcyF#H#5YcTlK({ zm34KUJ-&%aQfs=U){K#*K{Kn|^m_G_Sk1Bm&r|v9zPXyv|N^pN+hnge#mT98SZ9!HF&jJ&F8MePmlDWR)(9?H)^c!ko6PM2bUCwq*M1>vfesBd^Mp;H}1jgyA8+@8%o}`|BSwV{)`N@ zg|P=uPG8HaZJgPBDaCkYMZfV3JV@Yig{3mTj&IQmy<}lOv*q#Rbn)c}L(g~lW+wp# zoI54A36#FbHq0PRFXj60ok}SAVDv+Ipt$|X$%WnnS z)r7Vyr64t#Quygdl2{vZAV~}z-RTGOSU2wF7?uU5763{gS1?v)=z(7oa2ePZlw3V& z0>-{C84j4z>4%$hSY;UpE>z=`du%GhoED!m&}@?`bCfA795sHKlgQlpuM>*J*`Lc_ z02_9_lU$ONE_I!Fydm&X#5RT7L}KT9Ef6E+OhUs-bl{M#+DC zy7e(-eu+hcH*GSnD+z#;a0M4-+c&p#k!O78ZEaC&-hldn(+kZC$#j4i;R-5b4)3h{ z^pw;wlur_PzU(+wztJxQ`AnkZRu1pr<+5|Dw4$qcq!AqTJp;0%expLr`YdSI*XYb; z^&n+Q$-N?tl#fpJ+TqfuvF7eX-ow0!HrwzjH?dh&7rxFYj^({K^P55k@ZsjO|Kd~UAc_#8e{jTDFtYf|%Ai$;Mc}WC*u49S=-GAT8 zOd8iI6irbrpiXVRR4swW%*aHlcDoj z%reR}c^u{389&DCH#1g*-yl9tI9q2j_P0L*78<&)B=f($gC*f8GGxI?|E#pzTX65_ z{-H;XIXc^ktL;1sGyyZ6k{2d5>qm7}Z2y;!#S8ioqB{t zP}H@Y85QDsPx2h1r*awrkb1bnBQkY~xn{{>D3wi`Nx-tAUVcteW+e$gqO-V!wQ;-}Ly?g{`2l)T~k{{wiTdtmFu zQF6CYgw2-ey%SHM>S9aD9VPYXrwd8(K=(k7=DJ@lq!En^T(2g^fCyYg)-u=PL!9%q z!e^IF{Gxa-`{Z*v@2kZS+OBY12*ZK;e$6Rn{-f3-7q$Mrb8-uk0?mtLnnnBNHCrkq z0R~FMl3~r?Q=k^x=r+6Iv{3;etE7v6A=1-Sc@@G!QH+-96FhS{y=zh z6|hYyxdLfmzb8V?d;ovor%S2YLb~L3DegyHsn&l6(yj!oP2Wq^R&}%Ab7JPpb#nm= z@hq3e(*H9Db3Si*(&zYeH_9)R(??Q|eon!n?m^L_$F32@MNJnQn-atyGW%}fr`wvgP7DwN-)IyMMi14{#BMjx5EmqOUfw# zgx#nw1%3~L>W$WOOCGJ-vo1*II&pw?S^08W$4rw+>wVDTa2+}SD7&82YvSYcPWB8r zkyO&!CD&2f9YDeRK%mS-7Ef=+kH|k71#H;TLGteds-aI-ZgTioJaSO_K2W^|fbc#5 z7s4;^1B#=63jL_BFAeqq4sP}Wxf@1tO!q*x4Q%}%)wy0V*?dn&Wpk3$getiJDf}e~ z(5{j1he|hHrC0HG1Osp?C9g(4KyzBpEuML9kC#S~bB{oAOULK%&`HP)GfH9SkKnjX zy1wo-U468BPyfq)iuv*#jDw}yoO0X&2zL@DyOS8aS*;J}VQQB+3kFl~wH(&DQ|Jv4 zxRVrQA*2m%OipLk9vd&albG`X0UJk2K8h;6t(l3GH9uDp)eH@h<%?S&0B-^2->;*Y zgPoEWYmFK!eZR)DL78W@w0P>Scc*^{7R_V<8g3&mnE^oxD%Hugb9s9r%&!6q?zK76 zaA%4CPL-&!aJ!aOquv@;sEyMbtyOqbh@any&wACIF}a1U^5X$X)%0o3WdQ~5l5o*XR}QiQl0?=zG@+1Xox>DhR?=J$4XiP zVF(+>r2?TqmJo#5i$1YJkJ`^VB@+rs;5Mpph)~1n&Ii0B!N!hgLnM&!Y*q;R7Aj>* ze#eX9PYcWuP@XsjW+d*!CcsB3qvI{^du{t^Xy3ppiWp(^IP0A_OeOs7^Ba?|41W8D zgvY`sx8b1LFUBxL*m!|Z5^%&v_JJjkYrujZC=sZkc_3Up{RehB%mEORBH^hF_Yzsy z82>mAIy8oHGCq1ly8oA)46lap0G8lE?w568Qil7l>wugL_lx|0vHKUC{JQ@Sax(jV z@fRj#^ZzIRWPTe!Zs{nH&|pqNXa~X=XAiv9h%ww?UgbTEvw)TZp?5ZaAS)oWubqKR zhsRT(HvyUiG!1AfkRuQY2w~71dNv0_7@Of9=|McC6ZJ^PWFUk^FOvE|4nT;9uxMW* zeXc-eKEEN>T*L$;$e zk$rGh=v?SgL-9nqLSh&~8uWo2rlf}+kTq+TwCb516>^b6L-?;4 Tqy#C&pfZF)Wgt32Wgz?;p_pb- literal 0 HcmV?d00001 diff --git a/peer-review-template-rubric-guard/reports/rubric-guard-packet.json b/peer-review-template-rubric-guard/reports/rubric-guard-packet.json new file mode 100644 index 00000000..b2374649 --- /dev/null +++ b/peer-review-template-rubric-guard/reports/rubric-guard-packet.json @@ -0,0 +1,224 @@ +[ + { + "scenario": "Biology fast-track template misses rubric and privacy requirements", + "generatedAt": "2026-05-22T17:00:00Z", + "templateId": "tpl-biology-fast-track", + "discipline": "biology", + "decision": "hold-template", + "summary": { + "findingCount": 8, + "blockingFindingCount": 6, + "reviewFindingCount": 2 + }, + "findings": [ + { + "type": "missing-required-rubric-dimension", + "severity": "block", + "subject": "rigor", + "message": "Template is missing the required rigor rubric dimension." + }, + { + "type": "missing-required-rubric-dimension", + "severity": "block", + "subject": "reproducibility", + "message": "Template is missing the required reproducibility rubric dimension." + }, + { + "type": "score-scale-out-of-bounds", + "severity": "block", + "subject": "novelty", + "message": "novelty uses score scale -1..12, outside the supported 0..10 increasing range." + }, + { + "type": "evidence-anchor-missing", + "severity": "block", + "subject": "dataset", + "message": "No rubric dimension requires evidence anchored to dataset." + }, + { + "type": "evidence-anchor-missing", + "severity": "block", + "subject": "code", + "message": "No rubric dimension requires evidence anchored to code." + }, + { + "type": "evidence-anchor-missing", + "severity": "block", + "subject": "notebook", + "message": "No rubric dimension requires evidence anchored to notebook." + }, + { + "type": "anonymous-mode-privacy-gap", + "severity": "review", + "subject": "double-blind", + "message": "Anonymous or blind review mode does not hide both author and reviewer identities with a safe label policy." + }, + { + "type": "profile-publication-packet-incomplete", + "severity": "review", + "subject": "tpl-biology-fast-track", + "message": "Profile or citation-page publication packet must include rubric versioning and anonymous identity redaction." + } + ], + "curatorActions": [ + { + "type": "add-rubric-dimension", + "target": "rigor", + "reason": "Add a rigor scoring dimension before this peer-review template can affect reputation." + }, + { + "type": "add-rubric-dimension", + "target": "reproducibility", + "reason": "Add a reproducibility scoring dimension before this peer-review template can affect reputation." + }, + { + "type": "normalize-score-scale", + "target": "novelty", + "reason": "Use an increasing numeric score scale within 0..10 so reputation deltas are comparable." + }, + { + "type": "add-evidence-anchor", + "target": "dataset", + "reason": "Require at least one scoring dimension to cite dataset evidence before profile or timeline credit is computed." + }, + { + "type": "add-evidence-anchor", + "target": "code", + "reason": "Require at least one scoring dimension to cite code evidence before profile or timeline credit is computed." + }, + { + "type": "add-evidence-anchor", + "target": "notebook", + "reason": "Require at least one scoring dimension to cite notebook evidence before profile or timeline credit is computed." + }, + { + "type": "repair-review-mode-privacy", + "target": "tpl-biology-fast-track", + "reason": "Hide author and reviewer identities and replace real-name labels before publishing review receipts." + }, + { + "type": "complete-profile-publication-packet", + "target": "tpl-biology-fast-track", + "reason": "Include rubric version metadata and anonymous identity redaction before profile or citation publication." + } + ], + "templatePacket": { + "templateId": "tpl-biology-fast-track", + "discipline": "biology", + "decision": "hold-template", + "profileSafe": false, + "rubricScore": 0, + "requiredDimensions": [ + "clarity", + "rigor", + "novelty", + "reproducibility" + ], + "reviewedArtifactTypes": [ + "document", + "dataset", + "code", + "notebook" + ], + "publicationTargets": [ + "project-timeline", + "reviewer-profile" + ] + } + }, + { + "scenario": "Physics methods template needs reviewer declarations", + "generatedAt": "2026-05-22T17:10:00Z", + "templateId": "tpl-physics-methods", + "discipline": "physics", + "decision": "curator-review", + "summary": { + "findingCount": 2, + "blockingFindingCount": 0, + "reviewFindingCount": 2 + }, + "findings": [ + { + "type": "reviewer-declaration-incomplete", + "severity": "review", + "subject": "conflictOfInterest", + "message": "Reviewer declaration conflictOfInterest must be completed before reputation deltas are trusted." + }, + { + "type": "reviewer-declaration-incomplete", + "severity": "review", + "subject": "dataAccess", + "message": "Reviewer declaration dataAccess must be completed before reputation deltas are trusted." + } + ], + "curatorActions": [ + { + "type": "complete-reviewer-declaration", + "target": "conflictOfInterest", + "reason": "Require reviewer attestations before template output affects profiles or leaderboards." + }, + { + "type": "complete-reviewer-declaration", + "target": "dataAccess", + "reason": "Require reviewer attestations before template output affects profiles or leaderboards." + } + ], + "templatePacket": { + "templateId": "tpl-physics-methods", + "discipline": "physics", + "decision": "curator-review", + "profileSafe": true, + "rubricScore": 84, + "requiredDimensions": [ + "clarity", + "rigor", + "novelty", + "reproducibility" + ], + "reviewedArtifactTypes": [ + "document", + "code" + ], + "publicationTargets": [ + "project-timeline" + ] + } + }, + { + "scenario": "Social-science open review template is publication ready", + "generatedAt": "2026-05-22T17:20:00Z", + "templateId": "tpl-social-science-open-review", + "discipline": "social-sciences", + "decision": "publish-template", + "summary": { + "findingCount": 0, + "blockingFindingCount": 0, + "reviewFindingCount": 0 + }, + "findings": [], + "curatorActions": [], + "templatePacket": { + "templateId": "tpl-social-science-open-review", + "discipline": "social-sciences", + "decision": "publish-template", + "profileSafe": true, + "rubricScore": 100, + "requiredDimensions": [ + "clarity", + "rigor", + "novelty", + "reproducibility" + ], + "reviewedArtifactTypes": [ + "document", + "dataset", + "notebook" + ], + "publicationTargets": [ + "project-timeline", + "reviewer-profile", + "citation-page" + ] + } + } +] diff --git a/peer-review-template-rubric-guard/reports/rubric-guard-review.md b/peer-review-template-rubric-guard/reports/rubric-guard-review.md new file mode 100644 index 00000000..7d2fec88 --- /dev/null +++ b/peer-review-template-rubric-guard/reports/rubric-guard-review.md @@ -0,0 +1,59 @@ +# Peer Review Template Rubric Guard: tpl-biology-fast-track + +Discipline: biology +Decision: hold-template +Generated: 2026-05-22T17:00:00Z +Rubric score: 0 +Profile safe: false + +## Findings +- missing-required-rubric-dimension: rigor - Template is missing the required rigor rubric dimension. +- missing-required-rubric-dimension: reproducibility - Template is missing the required reproducibility rubric dimension. +- score-scale-out-of-bounds: novelty - novelty uses score scale -1..12, outside the supported 0..10 increasing range. +- evidence-anchor-missing: dataset - No rubric dimension requires evidence anchored to dataset. +- evidence-anchor-missing: code - No rubric dimension requires evidence anchored to code. +- evidence-anchor-missing: notebook - No rubric dimension requires evidence anchored to notebook. +- anonymous-mode-privacy-gap: double-blind - Anonymous or blind review mode does not hide both author and reviewer identities with a safe label policy. +- profile-publication-packet-incomplete: tpl-biology-fast-track - Profile or citation-page publication packet must include rubric versioning and anonymous identity redaction. + +## Curator Actions +- add-rubric-dimension: rigor - Add a rigor scoring dimension before this peer-review template can affect reputation. +- add-rubric-dimension: reproducibility - Add a reproducibility scoring dimension before this peer-review template can affect reputation. +- normalize-score-scale: novelty - Use an increasing numeric score scale within 0..10 so reputation deltas are comparable. +- add-evidence-anchor: dataset - Require at least one scoring dimension to cite dataset evidence before profile or timeline credit is computed. +- add-evidence-anchor: code - Require at least one scoring dimension to cite code evidence before profile or timeline credit is computed. +- add-evidence-anchor: notebook - Require at least one scoring dimension to cite notebook evidence before profile or timeline credit is computed. +- repair-review-mode-privacy: tpl-biology-fast-track - Hide author and reviewer identities and replace real-name labels before publishing review receipts. +- complete-profile-publication-packet: tpl-biology-fast-track - Include rubric version metadata and anonymous identity redaction before profile or citation publication. + +--- +# Peer Review Template Rubric Guard: tpl-physics-methods + +Discipline: physics +Decision: curator-review +Generated: 2026-05-22T17:10:00Z +Rubric score: 84 +Profile safe: true + +## Findings +- reviewer-declaration-incomplete: conflictOfInterest - Reviewer declaration conflictOfInterest must be completed before reputation deltas are trusted. +- reviewer-declaration-incomplete: dataAccess - Reviewer declaration dataAccess must be completed before reputation deltas are trusted. + +## Curator Actions +- complete-reviewer-declaration: conflictOfInterest - Require reviewer attestations before template output affects profiles or leaderboards. +- complete-reviewer-declaration: dataAccess - Require reviewer attestations before template output affects profiles or leaderboards. + +--- +# Peer Review Template Rubric Guard: tpl-social-science-open-review + +Discipline: social-sciences +Decision: publish-template +Generated: 2026-05-22T17:20:00Z +Rubric score: 100 +Profile safe: true + +## Findings +- None + +## Curator Actions +- No curator action required diff --git a/peer-review-template-rubric-guard/reports/summary.svg b/peer-review-template-rubric-guard/reports/summary.svg new file mode 100644 index 00000000..3e08bf25 --- /dev/null +++ b/peer-review-template-rubric-guard/reports/summary.svg @@ -0,0 +1,23 @@ + + + Peer Review Template Rubric Guard + Validates structured peer-review templates before profile, timeline, citation, or reputation updates + + + Held + 1 + + + + Curator Review + 1 + + + + Publish + 1 + + Findings: 10 | Blocking: 6 | Review: 4 | Curator actions: 10 + Checks: required rubric dimensions, score scales, evidence anchors, review-mode privacy, declarations, profile packets + Synthetic templates only. No live reviewers, profiles, projects, identities, credentials, or external services. + diff --git a/peer-review-template-rubric-guard/requirements-map.md b/peer-review-template-rubric-guard/requirements-map.md new file mode 100644 index 00000000..a0c24fee --- /dev/null +++ b/peer-review-template-rubric-guard/requirements-map.md @@ -0,0 +1,18 @@ +# Requirements Map + +Issue #15 asks for structured peer review, contribution credit, profile history, transparent reputation scoring, leaderboards, and badge systems. This slice focuses on the structured peer-review template gate that protects downstream reputation from malformed review inputs. + +| Issue requirement | Implementation coverage | +| --- | --- | +| Discipline-specific peer-review templates | `discipline` and template fixture scenarios model biology, physics, and social-science templates. | +| Optional scoring on clarity, rigor, novelty, reproducibility | Required rubric dimensions are enforced by `evaluatePeerReviewTemplateRubric`. | +| Inline comments on documents, datasets, code, and notebooks | `reviewedArtifactTypes` and `evidenceAnchors` require rubric dimensions to cite relevant artifacts before credit is computed. | +| Public, semi-private, anonymous, blind, and double-blind review modes | `visibilityMode` and `reviewMode` checks detect privacy gaps before profile or timeline publication. | +| Review history tracked on profiles and timelines | `publicationTargets` plus `profilePacket` ensure rubric versioning and anonymous identity redaction before publication. | +| Transparent reputation metrics | Findings and curator actions explain why a template is held, routed to review, or safe to publish. | +| Reviewer-ready demo | `demo.js` writes JSON, Markdown, SVG, and H.264 MP4 artifacts in `reports/`. | +| Safe local verification | `test.js` covers hold, curator review, and publish decisions without external calls or secrets. | + +## Distinctness + +This module avoids duplicating existing issue #15 slices by staying narrow: it does not implement a broad community ledger, endorsement ring guard, badge or leaderboard gate, review civility checker, timeliness module, recusal/COI guard, calibration bench, edit-history integrity guard, identity-leak-only guard, profile visibility guard, appeals queue, mentorship ledger, correction-impact ledger, or contributor credit attestation module. It validates review template structure before any downstream reputation logic runs. diff --git a/peer-review-template-rubric-guard/sample-data.js b/peer-review-template-rubric-guard/sample-data.js new file mode 100644 index 00000000..829a8450 --- /dev/null +++ b/peer-review-template-rubric-guard/sample-data.js @@ -0,0 +1,68 @@ +const scenarios = [ + { + name: 'Biology fast-track template misses rubric and privacy requirements', + generatedAt: '2026-05-22T17:00:00Z', + templateId: 'tpl-biology-fast-track', + discipline: 'biology', + reviewedArtifactTypes: ['document', 'dataset', 'code', 'notebook'], + visibilityMode: 'double-blind', + dimensions: [ + {name: 'clarity', weight: 0.3, scoreScale: {min: 0, max: 10}, evidenceAnchors: ['document']}, + {name: 'novelty', weight: 0.5, scoreScale: {min: -1, max: 12}, evidenceAnchors: ['document']}, + ], + reviewMode: { + authorIdentityHidden: true, + reviewerIdentityHidden: false, + anonymousLabelPolicy: 'real-name', + }, + reviewerDeclarations: {conflictOfInterest: true, methodExpertise: true, dataAccess: true}, + publicationTargets: ['project-timeline', 'reviewer-profile'], + profilePacket: {includesRubricVersion: true, redactsAnonymousIdentities: false}, + }, + { + name: 'Physics methods template needs reviewer declarations', + generatedAt: '2026-05-22T17:10:00Z', + templateId: 'tpl-physics-methods', + discipline: 'physics', + reviewedArtifactTypes: ['document', 'code'], + visibilityMode: 'semi-private', + dimensions: [ + {name: 'clarity', weight: 0.25, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'rigor', weight: 0.35, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document', 'code']}, + {name: 'novelty', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'reproducibility', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['code']}, + ], + reviewMode: { + authorIdentityHidden: false, + reviewerIdentityHidden: false, + anonymousLabelPolicy: 'visible-to-authors', + }, + reviewerDeclarations: {conflictOfInterest: false, methodExpertise: true, dataAccess: false}, + publicationTargets: ['project-timeline'], + profilePacket: {includesRubricVersion: true, redactsAnonymousIdentities: true}, + }, + { + name: 'Social-science open review template is publication ready', + generatedAt: '2026-05-22T17:20:00Z', + templateId: 'tpl-social-science-open-review', + discipline: 'social-sciences', + reviewedArtifactTypes: ['document', 'dataset', 'notebook'], + visibilityMode: 'public', + dimensions: [ + {name: 'clarity', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'rigor', weight: 0.35, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document', 'dataset']}, + {name: 'novelty', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'reproducibility', weight: 0.25, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['dataset', 'notebook']}, + ], + reviewMode: { + authorIdentityHidden: false, + reviewerIdentityHidden: false, + anonymousLabelPolicy: 'public-name', + }, + reviewerDeclarations: {conflictOfInterest: true, methodExpertise: true, dataAccess: true}, + publicationTargets: ['project-timeline', 'reviewer-profile', 'citation-page'], + profilePacket: {includesRubricVersion: true, redactsAnonymousIdentities: true}, + }, +]; + +module.exports = {scenarios}; diff --git a/peer-review-template-rubric-guard/test.js b/peer-review-template-rubric-guard/test.js new file mode 100644 index 00000000..03b9cbf7 --- /dev/null +++ b/peer-review-template-rubric-guard/test.js @@ -0,0 +1,110 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { + evaluatePeerReviewTemplateRubric, + buildRubricGuardPacket, +} = require('./index'); + +test('holds structured review templates that miss required dimensions and evidence anchors', () => { + const result = evaluatePeerReviewTemplateRubric({ + generatedAt: '2026-05-22T17:00:00Z', + templateId: 'tpl-biology-fast-track', + discipline: 'biology', + reviewedArtifactTypes: ['document', 'dataset', 'code', 'notebook'], + visibilityMode: 'double-blind', + dimensions: [ + {name: 'clarity', weight: 0.3, scoreScale: {min: 0, max: 10}, evidenceAnchors: ['document']}, + {name: 'novelty', weight: 0.5, scoreScale: {min: -1, max: 12}, evidenceAnchors: ['document']}, + ], + reviewMode: { + authorIdentityHidden: true, + reviewerIdentityHidden: false, + anonymousLabelPolicy: 'real-name', + }, + reviewerDeclarations: {conflictOfInterest: true, methodExpertise: true, dataAccess: true}, + publicationTargets: ['project-timeline', 'reviewer-profile'], + profilePacket: {includesRubricVersion: true, redactsAnonymousIdentities: false}, + }); + + assert.equal(result.decision, 'hold-template'); + assert.deepEqual( + result.findings.map((finding) => finding.type), + [ + 'missing-required-rubric-dimension', + 'missing-required-rubric-dimension', + 'score-scale-out-of-bounds', + 'evidence-anchor-missing', + 'evidence-anchor-missing', + 'evidence-anchor-missing', + 'anonymous-mode-privacy-gap', + 'profile-publication-packet-incomplete', + ] + ); + assert.equal(result.summary.blockingFindingCount, 6); + assert.match(result.curatorActions[0].reason, /rigor/i); +}); + +test('routes otherwise valid templates to curator review when reviewer declarations are incomplete', () => { + const result = evaluatePeerReviewTemplateRubric({ + generatedAt: '2026-05-22T17:10:00Z', + templateId: 'tpl-physics-methods', + discipline: 'physics', + reviewedArtifactTypes: ['document', 'code'], + visibilityMode: 'semi-private', + dimensions: [ + {name: 'clarity', weight: 0.25, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'rigor', weight: 0.35, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document', 'code']}, + {name: 'novelty', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'reproducibility', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['code']}, + ], + reviewMode: { + authorIdentityHidden: false, + reviewerIdentityHidden: false, + anonymousLabelPolicy: 'visible-to-authors', + }, + reviewerDeclarations: {conflictOfInterest: false, methodExpertise: true, dataAccess: false}, + publicationTargets: ['project-timeline'], + profilePacket: {includesRubricVersion: true, redactsAnonymousIdentities: true}, + }); + + assert.equal(result.decision, 'curator-review'); + assert.deepEqual( + result.findings.map((finding) => finding.type), + ['reviewer-declaration-incomplete', 'reviewer-declaration-incomplete'] + ); + assert.equal(result.summary.reviewFindingCount, 2); + assert.equal(result.templatePacket.profileSafe, true); +}); + +test('publishes complete rubric templates and builds a profile-safe review packet', () => { + const result = evaluatePeerReviewTemplateRubric({ + generatedAt: '2026-05-22T17:20:00Z', + templateId: 'tpl-social-science-open-review', + discipline: 'social-sciences', + reviewedArtifactTypes: ['document', 'dataset', 'notebook'], + visibilityMode: 'public', + dimensions: [ + {name: 'clarity', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'rigor', weight: 0.35, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document', 'dataset']}, + {name: 'novelty', weight: 0.2, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['document']}, + {name: 'reproducibility', weight: 0.25, scoreScale: {min: 1, max: 5}, evidenceAnchors: ['dataset', 'notebook']}, + ], + reviewMode: { + authorIdentityHidden: false, + reviewerIdentityHidden: false, + anonymousLabelPolicy: 'public-name', + }, + reviewerDeclarations: {conflictOfInterest: true, methodExpertise: true, dataAccess: true}, + publicationTargets: ['project-timeline', 'reviewer-profile', 'citation-page'], + profilePacket: {includesRubricVersion: true, redactsAnonymousIdentities: true}, + }); + const packet = buildRubricGuardPacket(result); + + assert.equal(result.decision, 'publish-template'); + assert.equal(result.summary.findingCount, 0); + assert.equal(result.templatePacket.templateId, 'tpl-social-science-open-review'); + assert.equal(result.templatePacket.rubricScore, 100); + assert.match(packet, /social-sciences/); + assert.match(packet, /No curator action required/); +});