{
  "skill_name": "devrel-hiring",
  "evals": [
    {
      "id": 1,
      "prompt": "I'm Priya, cofounder at Lattice Fold. We're pre-seed, 7 people, raised $1.4M, building a build-cache service for WASM toolchains. I spent six years at a big cloud vendor and I still have the Developer Advocate II job description we used there - it was anchored to our L4 engineering level, asked for sustained conference output in one lane (the candidate had to show two years of consistent talk history on a single technology), and offered $185k base plus stock plus a $20k annual travel budget. Can you adapt that doc for our first DevRel hire? We can pay about $110k plus 0.6% equity and we have maybe $4k for travel for the whole year. Sales are founder-led right now and what we really need is for developers to find us and get to a working build without talking to us.",
      "expected_output": "A scorecard and posting rebuilt from the early-startup company-type row rather than an edit of the big-tech description: generalist-across-four-pillars framing, a named funding driver in the mission line, quantified ranked outcomes sized to a $4k travel budget, weighted competencies, and a founder round added to the loop.",
      "files": [],
      "expectations": [
        "Rejects adapting the big-tech job description text and rebuilds the scorecard from the early-startup company-type row instead",
        "States the role is a generalist across all four pillars (advocacy, marketing, enablement, community) and self-directed, rather than deep in one lane",
        "Notes that levelling anchored to a big-tech engineering ladder does not transfer, and that 'senior' at this stage means founding generalist",
        "Sets the portfolio bar as breadth plus at least one end-to-end outcome the candidate owned alone, not sustained high-volume output in a single lane",
        "Names a funding driver explicitly in the mission line (for example developer adoption or developer enablement) rather than a generic 'grow the community' purpose",
        "Flags the $4k annual travel budget as incompatible with any conference-heavy outcome carried over from the big-tech description",
        "Produces between 3 and 8 outcomes, each quantified with a number and a time window, and ranked by importance",
        "Produces between 5 and 7 competencies, weighted for this company type and sub-role",
        "Adds a founder or exec round to the interview loop because this is an early-startup hire",
        "States the coding-depth bar at this stage is high because the same person builds the demos and the docs"
      ]
    },
    {
      "id": 2,
      "prompt": "Can you tighten up this job ad before I post it? Company is Helix Relay, we do event-stream tooling.\n\n'Developer Advocate (Associate) - $72,000\nRequirements: 5+ years shipping production systems in Go, Rust, TypeScript and Python. Demonstrated open-source contributions to a CNCF project. A published technical writing portfolio. Experience organising a conference track or meetup series. 10k+ combined social following across X, LinkedIn and YouTube. You'll write blog posts, speak at events, run our Discord, answer support questions and help sales on technical calls.'\n\nMostly I want the wording to sound warmer and less corporate.",
      "expected_output": "A refusal to merely reword: the posting is identified as the gatekeeper pattern, audited against the five-item self-audit checklist, and returned as requiring a structural rewrite with the requirement list cut to the actual pillar mix or the level re-priced.",
      "files": [],
      "expectations": [
        "Identifies the posting as the 'gatekeeper' failure pattern by name rather than treating it as a tone problem",
        "Attributes the gatekeeper/pit-trap taxonomy to Joe Nash and DevRelCon 2021",
        "Names the levelling inconsistency directly: an 'Associate' title at $72,000 against requirements implying a full senior engineering career",
        "Flags the '10k+ combined social following' requirement as off-core-skills emphasis rather than a fundamental of communicating with developers",
        "Flags the four-language production-engineering requirement as describing an impossible person rather than the role's actual pillar mix",
        "States the posting needs a rewrite before publishing, not after it fails to convert candidates",
        "Runs a tactics-versus-outcomes check and notes the activity list (write, speak, run Discord, answer support, help sales) traces to no stated outcome",
        "Cuts the requirement list down to what the role's pillar mix actually needs, instead of softening the wording of each bullet",
        "Recommends re-pricing the level against the real requirement, or cutting the requirement to match the stated pay",
        "Makes no assertion about what any jurisdiction's pay-transparency law currently requires without asking the user to confirm the hiring location's rule"
      ]
    },
    {
      "id": 3,
      "prompt": "Series A, 40 people, remote-first with an office in Denver. Here's the posting I drafted for our first ever DevRel person - does this read well?\n\n'Founding Developer Advocate. No formal DevRel background required - we'll figure this out together! You'll own our documentation, write and maintain our SDK samples, run the community Discord, take a shift on the support rota, join technical sales calls when needed, and build internal enablement training for our AEs. You'll wear a lot of hats and have enormous impact. Competitive salary and equity.'\n\nI'm pretty happy with the energy of it, just want a second opinion.",
      "expected_output": "The posting is named as the pit-trap pattern and returned with three structural requirements (resources, ladder, twelve-month goal) plus a real compensation stance, with postponing the hire offered as a legitimate outcome.",
      "files": [],
      "expectations": [
        "Identifies the posting as the 'pit trap' failure pattern by name",
        "Names the six functions loaded onto one person (docs, SDKs, community, support, sales engineering, internal training) as the pit-trap signature rather than as attractive breadth",
        "Requires the posting to state explicitly what resources the role gets",
        "Requires the posting to state a career ladder or progression path, or to say plainly that none exists yet",
        "Requires a stated twelve-month goal for the role",
        "Offers postponing the hire until resources, ladder and goal exist as a legitimate outcome, not only rewriting the text",
        "Rejects 'competitive salary and equity' and requires either a published range or an explicit statement of why the range is gated",
        "Does not assert what Colorado or any other named jurisdiction's pay-transparency law currently requires, and asks the user to confirm the current rule for the hiring location instead",
        "Points out that 'no formal DevRel background required' is internally inconsistent with the scope the posting loads onto the role",
        "Names the funding driver the role will be measured on, or flags that the posting does not state one"
      ]
    },
    {
      "id": 4,
      "prompt": "We're down to three candidates for a Developer Advocate opening at Quilt Metrics (usage-metering API, Series B). For the next stage I want a take-home: write a 1,200-word tutorial using our API and record a 5-minute Loom walking through it. Realistically 6-8 hours of work. Unpaid obviously, everyone in this industry does this. Two of the three already have public blogs and a couple of conference talks between them; the third is a career-changer with basically nothing public. How should I brief the take-home and what rubric should I score it on?",
      "expected_output": "The unpaid take-home is refused for the two candidates with public work, a six-signal portfolio evaluation is run first, and any surviving work sample is paid, scored on a frozen rubric, and applied identically to every candidate.",
      "files": [],
      "expectations": [
        "Refuses to design an unpaid spec-work take-home for the two candidates who already have public writing and speaking artefacts",
        "States that a portfolio evaluation runs before any new work is requested from a candidate",
        "Offers a paid live working session as the alternative where a work sample is genuinely needed",
        "States that any work sample, if run, must be paid and scored against a fixed rubric, never treated as free trial work",
        "Evaluates the portfolio-rich candidates against the six-signal rubric: public writing, public speaking, code in public, community evidence, teaching artefact, developer-voice artefact",
        "Notes that judging the existing public work first can remove the need for the content take-home entirely",
        "Does not accept 'everyone in this industry does this' as a reason to keep the exercise unpaid",
        "Requires the same exercise and the same rubric to apply to every candidate for the role, rather than running a different stage for the career-changer alone",
        "Recommends freezing the rubric so every candidate for the role is scored against the same bar",
        "Scores each of the six signals on a 0-3 scale where 0 is absent, 1 is claimed but unevidenced, 2 is evidenced, and 3 is evidenced with a stated outcome"
      ]
    },
    {
      "id": 5,
      "prompt": "Need a decision by tomorrow on who we advance for our product-focused Developer Advocate role. Growth-stage, product-led, everything is self-serve API signup.\n\nMarco: 47k followers on X, posts daily, mostly about his own consulting practice. Two Medium posts behind a paywall, and he mentioned a ghostwriter drafted them. No public repos I could find. Claims he 'drove 4 million impressions' at his last company - when I asked what that converted to he changed the subject and then said impressions are a vanity metric anyway. Nine years in software sales engineering.\n\nFen: finished a bootcamp 14 months ago. No blog, no talks. Three small public repos, I cloned one and it ran first try with a clear README. Filed 11 issues on our own GitHub last year, one of which turned into a full docs rewrite on our side. Extremely active in our Discord, answers other people's questions.\n\nGut says Marco because of the reach. Who do we advance?",
      "expected_output": "Both candidates scored across all six signals on the 0-3 scale, weighted toward code in public and developer-voice artefact for the product-focused sub-role, with Marco's negative signals named explicitly and a paid, scored work sample designed for Fen.",
      "files": [],
      "expectations": [
        "Scores both candidates across all six signals (public writing, public speaking, code in public, community evidence, teaching artefact, developer-voice artefact) on the 0-3 scale",
        "Weights the scoring toward code in public and the developer-voice artefact because the target sub-role is a product-focused advocate",
        "Names Marco's follower count with no substance behind it as a negative signal explicitly, not merely as a missing strength",
        "Names the ghostwritten, paywalled content Marco cannot defend in conversation as a negative signal",
        "Names Marco's shifting story on the impressions figure as the exaggerated-or-dismissed-metrics negative signal",
        "Credits Fen's 11 filed issues, and the one that produced a docs rewrite, as a developer-voice artefact",
        "Credits Fen's runnable public repos as code in public rather than discounting them as bootcamp projects",
        "States that a product-focused advocate is screened on a recent developer background, a stricter bar than the outreach-focused variant",
        "Designs a scored, paid work sample for Fen matched to the product-focused sub-role, such as reviewing an anonymised support thread and proposing the SDK or docs fix",
        "Marks the score bands used as a working baseline rather than presenting them as a published industry standard"
      ]
    },
    {
      "id": 6,
      "prompt": "Hiring a Community Manager for our Discord and forum - open-source distributed database, 35k Discord members, lots of heated threads about our licence change last year. I don't want to get this wrong so I want to run the full gauntlet: technical self-rating questionnaire first as a filter, then a coding take-home, a content take-home, a presentation round, three conversational rounds, a DevRel-opinion round, and a founder round at the end. We're a 9-person team and honestly I can spare about 3 hours of interviewer time per candidate in total. What order should these go in?",
      "expected_output": "Stages deleted rather than reordered: a 2-3 stage loop built from portfolio evaluation plus conflict-handling conversational rounds, with the coding take-home removed as non-discriminating and the self-rating questionnaire demoted from gate to follow-up targeting.",
      "files": [],
      "expectations": [
        "Deletes interview stages rather than only reordering the eight the user listed",
        "Cuts the loop to 2-3 stages for a community-manager hire",
        "Removes the coding take-home, stating it does not discriminate for this sub-role",
        "Keeps the portfolio evaluation and places it before any new work is requested of the candidate",
        "Keeps a conversational round with an explicit conflict-handling or de-escalation emphasis",
        "States there is no standard DevRel interview loop, so formats are chosen against this role's pillar mix rather than run by default",
        "Recommends skipping a live work sample entirely rather than running one unscored under the three-hour ceiling",
        "States that a technical self-rating questionnaire must never gate or veto a candidate on its own, and is used to target follow-up questions in a later round",
        "Screens the community-manager role on adjacent people-field experience and coaching or conflict-resolution communication rather than on hands-on coding depth",
        "Includes at least one DevRel-opinion question in whatever loop survives the cut"
      ]
    },
    {
      "id": 7,
      "prompt": "Our interview process at Brightwell: after all the rounds, the four interviewers get in a room for 30 minutes, talk the candidate through, and land on hire/no-hire by consensus. Works fine for our backend engineers. But we just lost a Developer Advocate candidate we all liked to a competing offer, and the debrief before that one was a mess - two people visibly flipped their opinion after the hiring manager spoke first, and then nobody wanted to be the holdout. Help me run a tighter debrief meeting.",
      "expected_output": "Scoring moved out of the debrief entirely: independent, evidence-noted scores submitted before any discussion, combined mechanically against pre-set scorecard weights, with the debrief limited to calibrating interpretation.",
      "files": [],
      "expectations": [
        "Moves scoring out of the debrief: every interviewer submits an independent score before any discussion happens",
        "Requires a written evidence note per competency alongside each submitted score",
        "Combines scores mechanically against pre-set competency weights rather than reaching a consensus verdict in the room",
        "States that the debrief calibrates interpretation of already-submitted scores and never originates a score",
        "Identifies the two interviewers flipping after the hiring manager spoke as the failure the change prevents, rather than a meeting-facilitation problem to coach around",
        "Requires competency weights to be set on the scorecard before the loop runs, not after candidates are seen",
        "Requires the same question set to be used for every candidate for the role",
        "Requires every interview question to map to a named scorecard competency",
        "Notes that the structured, independently-scored, mechanically-combined result is a general employment-selection finding rather than a DevRel-specific one",
        "Does not propose a blended single gut score or a group-consensus rating as the output of the debrief"
      ]
    },
    {
      "id": 8,
      "prompt": "Our first Developer Advocate starts March 2nd at Overtone - API observability, Series B, product-led, about 400 self-serve teams on the platform. I've got a 30-60-90 doc half written and my VP wants it tightened before Friday. Right now it says: month 1 'onboard, meet the team, read the docs, get set up with all our tools'; month 2 'start posting on the company blog and attend a conference'; month 3 'own the community'. Can you clean this up and make it sound more rigorous?",
      "expected_output": "A ramp plan anchored on artefacts rather than activities: a day-30 friction log with severity and proposed fix per point, a day-60 paired shipped artefact plus a landed friction-log fix and a first metric attempt, and day-90 self-directed ownership with a written funding-driver metric plan and a joint written decision.",
      "files": [],
      "expectations": [
        "Replaces the month-1 onboarding list with a shipped friction log as the day-30 deliverable",
        "Specifies that the friction log carries a severity and a proposed fix for each point",
        "Sets a day-60 milestone of one shipped artefact in the target pillar, paired with an existing team member",
        "Requires at least one friction-log fix from day 30 to land by day 60, driven through docs or engineering by the new hire",
        "Requires a first attempt at the metric the role's funding driver is measured on by day 60, even as a rough baseline number",
        "Sets a day-90 milestone of self-directed ownership of one recurring program slice, without a pairing partner",
        "Requires a written plan by day 90 for the funding-driver metric, covering what good looks like in two quarters and how it will be tracked",
        "Requires a joint, written manager and new-hire decision at day 90 on continuing at the current scope, adjusting it, or escalating a concern",
        "Treats the three missed milestones as an explicit decision trigger rather than an assumed extended grace period",
        "Replaces 'attend a conference' with a shipped deliverable, since attendance is an activity and not a milestone"
      ]
    },
    {
      "id": 9,
      "prompt": "I run the ecosystem team at the Torrent Mesh Foundation - vendor-neutral, six corporate members, we steward a networking project. We want to bring on 8-10 ambassadors this quarter. I already wrote the requirement list by copying our salaried Developer Advocate posting: 4+ years experience, own the conference schedule for their region, respond to community questions within 24 hours, and report to me in a weekly 1:1. Where should I post it to get applicants? I was going to put it on LinkedIn and Indeed and maybe boost it.",
      "expected_output": "The paid-versus-unpaid question is settled first, the salaried requirement list is refused for an ambassador program, and sourcing is redirected from generic boards to community scouting, organic-content scouting, referrals and niche DevRel boards.",
      "files": [],
      "expectations": [
        "Asks whether the ambassador role is paid or unpaid before accepting the copied requirement list",
        "States that foundation ambassador programs are frequently unpaid and gated on measurable contribution rather than communication polish",
        "Refuses to carry the salaried Developer Advocate requirement list into an unpaid ambassador program",
        "Flags the weekly reporting line and the 24-hour response obligation as employment-shaped obligations inappropriate for an unpaid program",
        "Cites a foundation ambassador program that is explicitly not a paid position and selects on a minimum contribution score, such as the CNCF Ambassador program",
        "Treats a salaried vendor-neutral foundation advocate posting and an unpaid ambassador program as two different postings, never one template",
        "States that generic job boards such as LinkedIn and Indeed underperform here, because the pool that both codes credibly and communicates publicly is small and already visible through other channels",
        "Recommends scouting already-engaged contributors inside the project's own community venues and approaching them directly",
        "Notes that technology-specific Slack and Discord communities generally forbid unsolicited direct messages, so a post in a dedicated jobs channel respects each community's own rules",
        "Names niche DevRel job boards as preferable to generic boards because their listings are pre-filtered to companies already running a DevRel function"
      ]
    }
  ],
  "trigger_queries": [
    { "query": "how do I hire a developer advocate", "should_trigger": true },
    { "query": "write me a job posting for a developer advocate", "should_trigger": true },
    { "query": "what should our DevRel interview loop look like", "should_trigger": true },
    { "query": "we need a scorecard for our first devrel hire", "should_trigger": true },
    { "query": "is this candidate's portfolio good enough to advance to the next round", "should_trigger": true },
    { "query": "design a take-home exercise for a developer advocate candidate", "should_trigger": true },
    { "query": "our developer evangelist job ad isn't converting, can you look at it", "should_trigger": true },
    { "query": "what questions should I ask a community manager candidate", "should_trigger": true },
    { "query": "30-60-90 plan for a new developer advocate starting next month", "should_trigger": true },
    { "query": "how do I tell whether a devrel candidate is any good", "should_trigger": true },
    { "query": "we're hiring our first person to do docs, talks and community - what do we even call the role", "should_trigger": true },
    { "query": "what's a fair interview process for a developer educator", "should_trigger": true },
    { "query": "should I ask a candidate to write us a blog post as a test", "should_trigger": true },
    { "query": "how many rounds should we run for a DX engineer hire", "should_trigger": true },
    { "query": "requirements list for a founding developer advocate", "should_trigger": true },
    { "query": "help me build a rubric to evaluate developer advocate candidates", "should_trigger": true },
    { "query": "we keep getting applicants who can talk but can't code - fix our screen", "should_trigger": true },
    { "query": "where do I find developer advocate candidates", "should_trigger": true },
    { "query": "our devrel job description reads like we want a unicorn", "should_trigger": true },
    { "query": "what should the first 90 days look like for a developer advocate", "should_trigger": true },
    { "query": "I need to write a JD for someone to run our developer community", "should_trigger": true },
    { "query": "how should four interviewers score a devrel candidate", "should_trigger": true },
    { "query": "what does 'senior developer advocate' actually mean when we're only 8 people", "should_trigger": true },
    { "query": "evaluate this developer advocate resume for us", "should_trigger": true },
    { "query": "should we put a salary range in our devrel posting", "should_trigger": true },
    { "query": "planning an interview panel for a developer relations manager", "should_trigger": true },
    { "query": "we want ambassadors for our open source project, how do we recruit them", "should_trigger": true },
    { "query": "what outcomes should I put on a devrel role's scorecard", "should_trigger": true },
    { "query": "reusing my old developer advocate JD from my last company at a 6-person startup, is that ok", "should_trigger": true },
    { "query": "designing a paid work sample for someone new to devrel", "should_trigger": true },
    { "query": "which interview formats actually predict a good developer advocate", "should_trigger": true },
    { "query": "how do I ramp a new community manager in their first quarter", "should_trigger": true },
    { "query": "our devrel hire failed at 6 months, what should we have tested for", "should_trigger": true },
    { "query": "what red flags should I look for in a developer advocate's public work", "should_trigger": true },
    { "query": "we need someone to do developer marketing and I don't know what to screen on", "should_trigger": true },
    { "query": "turn this list of responsibilities into a proper role definition for our advocate hire", "should_trigger": true },
    { "query": "briefing an external recruiter on a developer relations opening", "should_trigger": true },
    { "query": "onboarding plan for our new developer experience hire", "should_trigger": true },
    { "query": "how do I compare two devrel candidates with completely different backgrounds", "should_trigger": true },
    { "query": "should this role need to ship SDK samples or is speaking well enough", "should_trigger": true },
    { "query": "we only have 3 hours of interviewer time per candidate, what do we cut from the loop", "should_trigger": true },
    { "query": "how do I prep for a developer advocate interview next week", "should_trigger": false },
    { "query": "review my devrel portfolio before I start applying", "should_trigger": false },
    { "query": "I want to break into developer relations from backend engineering", "should_trigger": false },
    { "query": "should I take this developer advocate offer or stay where I am", "should_trigger": false },
    { "query": "how do I negotiate a devrel salary", "should_trigger": false },
    { "query": "what should be on my resume for a community manager role", "should_trigger": false },
    { "query": "am I ready to apply for a senior developer advocate position", "should_trigger": false },
    { "query": "should DevRel report to marketing or to product", "should_trigger": false },
    { "query": "how big should our devrel team be at series B", "should_trigger": false },
    { "query": "centralized or embedded devrel org, which shape fits us", "should_trigger": false },
    { "query": "who should own docs, devrel or engineering", "should_trigger": false },
    { "query": "how do we make our engineering brand attractive to senior engineers", "should_trigger": false },
    { "query": "what should our careers page say to developers", "should_trigger": false },
    { "query": "should we start an engineering blog to attract engineers", "should_trigger": false },
    { "query": "how do I measure devrel impact for my board deck", "should_trigger": false },
    { "query": "what devrel metrics should I report monthly", "should_trigger": false },
    { "query": "set up event tracking across our docs and blog", "should_trigger": false },
    { "query": "how much should we spend on devrel next year", "should_trigger": false },
    { "query": "what should our developer relations strategy be for next year", "should_trigger": false },
    { "query": "split our devrel budget across events and content", "should_trigger": false },
    { "query": "design a perks ladder for our developer champions program", "should_trigger": false },
    { "query": "what obligations should champions have in our ambassador program", "should_trigger": false },
    { "query": "should we start a Discord or a forum for our users", "should_trigger": false },
    { "query": "write a code of conduct for our developer community", "should_trigger": false },
    { "query": "how do I measure the health of our developer community", "should_trigger": false },
    { "query": "how do we get more first-time contributors to our repo", "should_trigger": false },
    { "query": "who decides what gets merged in our open source project", "should_trigger": false },
    { "query": "help me write a conference talk proposal", "should_trigger": false },
    { "query": "audit our quickstart guide", "should_trigger": false },
    { "query": "turn these commits into release notes", "should_trigger": false },
    { "query": "plan our devrel content calendar for Q3", "should_trigger": false },
    { "query": "map the developer journey for our platform team segment", "should_trigger": false },
    { "query": "which conferences should we sponsor this year", "should_trigger": false },
    { "query": "write a case study about how Northwind uses our API", "should_trigger": false },
    { "query": "write a job description for a senior backend engineer", "should_trigger": false },
    { "query": "design an interview loop for a product manager", "should_trigger": false },
    { "query": "build a hiring scorecard for our first sales rep", "should_trigger": false },
    { "query": "how do I run a technical coding interview", "should_trigger": false },
    { "query": "write our engineering onboarding checklist", "should_trigger": false },
    { "query": "recruiting strategy for a data science team", "should_trigger": false },
    { "query": "what questions should I ask in a reference check", "should_trigger": false }
  ]
}