runpod/comfyui/workflows/text-to-music/diffrhythm-reference-based-v1.json

{
  "last_node_id": 4,
  "last_link_id": 3,
  "nodes": [
    {
      "id": 1,
      "type": "LoadAudio",
      "pos": [100, 100],
      "size": [300, 100],
      "flags": {},
      "order": 0,
      "mode": 0,
      "outputs": [
        {
          "name": "AUDIO",
          "type": "AUDIO",
          "links": [1]
        }
      ],
      "properties": {},
      "widgets_values": [
        "reference_audio.wav"
      ],
      "title": "Load Reference Audio"
    },
    {
      "id": 2,
      "type": "DiffRhythmReferenceBasedGeneration",
      "pos": [500, 100],
      "size": [400, 350],
      "flags": {},
      "order": 1,
      "mode": 0,
      "inputs": [
        {
          "name": "reference_audio",
          "type": "AUDIO",
          "link": 1
        }
      ],
      "outputs": [
        {
          "name": "AUDIO",
          "type": "AUDIO",
          "links": [2]
        }
      ],
      "properties": {},
      "widgets_values": [
        "Energetic rock music with driving guitar riffs and powerful drums",
        95.0,
        5.0,
        0.7,
        456,
        "cfm_model_v1_2",
        "auto"
      ],
      "title": "DiffRhythm Reference-Based Generation"
    },
    {
      "id": 3,
      "type": "PreviewAudio",
      "pos": [1000, 100],
      "size": [300, 100],
      "flags": {},
      "order": 2,
      "mode": 0,
      "inputs": [
        {
          "name": "audio",
          "type": "AUDIO",
          "link": 2
        }
      ],
      "properties": {},
      "title": "Preview Generated Audio"
    },
    {
      "id": 4,
      "type": "SaveAudio",
      "pos": [1000, 250],
      "size": [300, 100],
      "flags": {},
      "order": 3,
      "mode": 0,
      "inputs": [
        {
          "name": "audio",
          "type": "AUDIO",
          "link": 3
        }
      ],
      "properties": {},
      "widgets_values": [
        "diffrhythm_reference_output"
      ],
      "title": "Save Audio"
    }
  ],
  "links": [
    [1, 1, 0, 2, 0, "AUDIO"],
    [2, 2, 0, 3, 0, "AUDIO"],
    [3, 2, 0, 4, 0, "AUDIO"]
  ],
  "groups": [],
  "config": {},
  "extra": {
    "workflow_info": {
      "name": "DiffRhythm Reference-Based Generation v1",
      "description": "Generate new music based on a reference audio file while following text prompt guidance",
      "version": "1.0.0",
      "author": "valknar@pivoine.art",
      "category": "text-to-music",
      "tags": ["diffrhythm", "music-generation", "reference-based", "style-transfer"],
      "requirements": {
        "custom_nodes": ["ComfyUI_DiffRhythm"],
        "models": ["ASLP-lab/DiffRhythm-1_2", "ASLP-lab/DiffRhythm-vae", "OpenMuQ/MuQ-MuLan-large", "OpenMuQ/MuQ-large-msd-iter", "FacebookAI/xlm-roberta-base"],
        "vram_min": "14GB",
        "vram_recommended": "18GB",
        "system_deps": ["espeak-ng"]
      },
      "usage": {
        "reference_audio": "Path to reference audio file (WAV, MP3, or other supported formats)",
        "prompt": "Text description guiding the style and characteristics of generated music",
        "duration": "Fixed at 95 seconds for DiffRhythm 1.2 model",
        "guidance_scale": "Controls how closely output follows the prompt (1.0-10.0, default: 5.0)",
        "reference_strength": "How much to follow the reference audio (0.0-1.0, default: 0.7)",
        "seed": "Random seed for reproducibility (default: 456)",
        "model": "cfm_model_v1_2 (DiffRhythm 1.2)",
        "device": "auto (automatic GPU selection)"
      },
      "use_cases": [
        "Style transfer: Apply the style of reference music to new prompt",
        "Variations: Create variations of existing compositions",
        "Genre transformation: Transform music to different genre while keeping structure",
        "Mood adaptation: Change the mood/emotion while maintaining musical elements"
      ],
      "notes": [
        "This workflow combines reference audio with text prompt guidance",
        "Higher reference_strength (0.8-1.0) = closer to reference audio",
        "Lower reference_strength (0.3-0.5) = more creative interpretation",
        "Reference audio should ideally be similar duration to target (95s)",
        "Can use any format supported by ComfyUI's audio loader",
        "PLACEHOLDER: Actual node names and parameters need to be updated after ComfyUI_DiffRhythm installation"
      ]
    }
  },
  "version": 0.4
}