A/B test post variantswith Jev before publishing

const OPENROUTER = 'https://openrouter.ai/api/v1/chat/completions';

export async function writeVariants(brief: string, n: number): Promise<string[]> {
  const res = await fetch(OPENROUTER, {
    method: 'POST',
    headers: {
      Authorization: 'Bearer ' + process.env.OPENROUTER_API_KEY,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      model: 'anthropic/claude-sonnet-4.5',
      // Temperature matters more here than anywhere else in the pipeline:
      // five near-identical variants give the scorer nothing to rank.
      temperature: 1,
      messages: [
        {
          role: 'system',
          content:
            'You write posts for X. One post per line, nothing else. ' +
            'Each line must take a genuinely different angle on the brief, ' +
            'not a reworded version of the same one.',
        },
        { role: 'user', content: 'Write ' + n + ' posts about: ' + brief },
      ],
    }),
  });

  const data = await res.json();
  return String(data.choices[0].message.content)
    .split('\n')
    .map((line: string) => line.trim())
    .filter(Boolean)
    .slice(0, n);
}

import { TypeSafeClient, noul, score } from '@typesafe-ai/sdk';

const client = new TypeSafeClient({ timeout: 4000 });

const QUESTIONS = {
  strength: score(
    'How far will this post travel compared to a typical post from the same author?',
    [
      'Flops. Gets less attention than a typical post from the same account.',
      'Typical. Performs about the same as the account usually does.',
      'Above typical. Noticeably more reach and replies than usual.',
      'Strong. Several times the usual reach. Spreads past the existing audience.',
      'Breakout. Orders of magnitude beyond usual.',
    ]
  ),
  slop: noul('This reads like it was generated by an AI rather than written by a person.'),
  bait: noul('This post explicitly asks the reader for likes, reposts, replies, follows, or bookmarks.'),
} as const;

export interface Ranked {
  text: string;
  score: number;                              // 0 to 4
  confidence: number;                         // 0 to 1
  distribution: Record<string, number>;       // one entry per rubric level
  slop: number;
  bait: number;
}

export async function rank(variants: string[]): Promise<Ranked[]> {
  // One request per variant, all in flight together. Parallel requests finish
  // in about the time the slowest one takes.
  const scored = await Promise.all(
    variants.map(async (text): Promise<Ranked> => {
      const { answers } = await client.systemOne({
        state: { draft_post: text, platform: 'X (Twitter)' },
        questions: QUESTIONS,
      });
      return {
        text,
        score: answers.strength.score,
        confidence: answers.strength.confidence,
        distribution: answers.strength.probabilities as Record<string, number>,
        slop: answers.slop.noul,
        bait: answers.bait.noul,
      };
    })
  );

  return scored.sort((a, b) => b.score - a.score);
}

import type { Ranked } from './rank';

// A gap smaller than this is not a result, it is rubric noise.
const MEANINGFUL_GAP = 0.25;
const MIN_CONFIDENCE = 0.6;

export function pick(ranked: Ranked[]) {
  const eligible = ranked.filter((v) => v.bait < 0.3 && v.slop < 0.5);
  if (eligible.length === 0) return { action: 'regenerate', reason: 'every variant failed a gate' };

  const [first, second] = eligible;

  // The winner is only a winner if the model was sure about it.
  if (first.confidence < MIN_CONFIDENCE) {
    return { action: 'review', winner: first, reason: 'flat distribution' };
  }

  // Two variants inside the noise floor are a tie, not a ranking. Ship the
  // shorter one and stop pretending the model chose.
  if (second && first.score - second.score < MEANINGFUL_GAP) {
    const shorter = first.text.length <= second.text.length ? first : second;
    return { action: 'publish', winner: shorter, reason: 'tie broken on length' };
  }

  return { action: 'publish', winner: first, reason: 'clear winner' };
}

const variants = await writeVariants('we cut our p99 from 1.2s to 180ms', 5);
const ranked = await rank(variants);
const decision = pick(ranked);

if (decision.action === 'publish') {
  const res = await fetch('https://opentweet.io/api/v1/posts', {
    method: 'POST',
    headers: {
      Authorization: 'Bearer ' + process.env.OPENTWEET_API_KEY,
      'Content-Type': 'application/json',
      'Idempotency-Key': 'p99-post-2026-09-22',
    },
    body: JSON.stringify({
      text: decision.winner.text,
      platforms: ['x', 'bluesky', 'linkedin'],
      category: 'Engineering',
      scheduled_date: '2026-09-22T14:00:00Z',
    }),
  });

  const data = await res.json();
  if (!res.ok) throw new Error((data.code || res.status) + ': ' + data.error);
  console.log(data.posts[0].id, data.posts[0].status);
}

// Keep the losers. A week of (variant, score, what actually happened) rows is
// the only thing that will ever tell you whether the ranking was any good.