Collapse

Introduction to Redpanda Connect

Redpanda Connect is a declarative connector framework that solves a wide range of data engineering problems with simple, chained, stateless processing steps. It implements transaction-based resiliency with back pressure, so when connecting to at-least-once sources and sinks it’s able to guarantee at-least-once delivery without needing to persist messages during transit.

It’s simple to deploy, comes with a wide range of connectors, and is totally data agnostic, making it easy to drop into your existing infrastructure. Redpanda Connect has functionality that overlaps with integration frameworks, log aggregators and ETL workflow engines, and can therefore be used to complement these traditional data engineering tools or act as a simpler alternative.

Next steps

Was this helpful?

group Ask in the community

mail Share your feedback

group_add Make a contribution

`; /** * For API endpoint results we show a tiny example to reduce clicks. * This is pure UI enrichment: not required by Algolia. */ const hasEndpoint = !!(preview && (preview.method || preview.path)); let endpointLine = ''; let curlExample = ''; if (hasEndpoint) { const method = (preview.method || 'GET').toUpperCase(); const path = preview.path || ''; endpointLine = `${method} ${path}`; let origin = ''; try { origin = new URL(preview.url || '').origin; } catch (e) { /* no-op */ } curlExample = `curl -X ${method} '${origin ? origin + path : path}'`; } // Integrate AI assistant with the current query. const askAIButton = currentQuery ? html`` : ''; // Compose a “View all results” link that respects product + version context. const componentTitle = 'Connect'; const productParams = (() => { if (componentTitle === 'Cloud') return '&product[0]=Cloud&product[1]=Connect'; if (componentTitle === 'Self-Managed') return '&product[0]=Self-Managed&product[1]=Connect'; return ''; })(); const viewAllLink = currentQuery ? html` View all results ` : ''; // Layout: header (filters/AI), results list, right-side preview, footer tips. render( html`

${dropdown} ${askAIButton}

${children}

${ preview ? html`

${ preview.breadcrumbs ? html`

{ event.stopPropagation(); // Click analytics requires clickAnalytics: true and a queryID. // Docs: https://www.algolia.com/doc/api-reference/api-parameters/clickAnalytics/ aa('clickedObjectIDsAfterSearch', { eventName: 'Preview Selected', index: state.context.preview.__autocomplete_indexName, queryID: state.context.preview.__autocomplete_queryID, objectIDs: [state.context.preview.objectID], positions: [state.activeItemId + 1], // positions start at 1 }); }} href="${breadcrumb.u}" > ${breadcrumb.t}

` : '' }

${components.Highlight({ hit: preview, attribute: 'title' })}

${ preview.intro ? components.Highlight({ hit: preview, attribute: 'intro' }) : preview.description ? components.Highlight({ hit: preview, attribute: 'description' }) : '' }

${ preview.api ? html`

${preview.api}

` : '' } ${ hasEndpoint ? html`

Endpoint

${endpointLine}

Example

${curlExample}

` : '' } ${ preview.image ? html`

${ SUPPORTED_PREVIEW_TYPES.includes((preview.type || '').toLowerCase()) ? html` { event.stopPropagation(); aa('clickedObjectIDsAfterSearch', { eventName: 'Preview Selected', index: state.context.preview.__autocomplete_indexName, queryID: state.context.preview.__autocomplete_queryID, objectIDs: [state.context.preview.objectID], positions: [state.activeItemId + 1], }); }} href="${preview.url || preview.objectID}" > ${preview.title || ''}

` : html` { event.stopPropagation(); aa('clickedObjectIDsAfterSearch', { eventName: 'Preview Selected', index: state.context.preview.__autocomplete_indexName, queryID: state.context.preview.__autocomplete_queryID, objectIDs: [state.context.preview.objectID], positions: [state.activeItemId + 1], }); }} target="_blank" rel="noopener noreferrer" href="${preview.url || preview.objectID}" > ${preview.title || ''}

` }

` : '' }

`, root ); }, /** * Define our data sources: * - "filters": facet values for the _tags facet (used by the tags plugin) * - "docs": actual content/hits * Sources overview: https://www.algolia.com/doc/ui-libraries/autocomplete/core-concepts/sources/ */ getSources({ query, state }) { const tagsByFacet = groupBy(state.context.tagsPlugin.tags, (tag) => tag.facet); return [ { sourceId: 'filters', getItems() { // Fetch facet values for _tags to populate the checkbox menu. // getAlgoliaFacets: https://www.algolia.com/doc/ui-libraries/autocomplete/api-reference/sources/#param-getalgoliafacets return getAlgoliaFacets({ searchClient, queries: [ { indexName: 'redpanda', facet: '_tags', params: { facetQuery: '', // https://www.algolia.com/doc/api-reference/api-parameters/facetQuery/ maxFacetHits: 50, // https://www.algolia.com/doc/api-reference/api-parameters/maxFacetHits/ }, }, ], transformResponse({ facetHits }) { // We return raw server facets; union with known+selected happens in render(). const server = (facetHits[0] || []).map(h => ({ label: h.label, facet: '_tags' })); return server; }, }); }, templates: { header() { return null; }, item() { return null; }, }, }, { sourceId: 'docs', getItems() { // getAlgoliaResults: https://www.algolia.com/doc/ui-libraries/autocomplete/api-reference/sources/#param-getalgoliaresults return getAlgoliaResults({ searchClient, queries: [ { indexName: 'redpanda', query, params: { clickAnalytics: true, // https://www.algolia.com/doc/api-reference/api-parameters/clickAnalytics/ hitsPerPage: 10, // https://www.algolia.com/doc/api-reference/api-parameters/hitsPerPage/ attributesToSnippet: ['*:25'], // https://www.algolia.com/doc/api-reference/api-parameters/attributesToSnippet/ snippetEllipsisText: '…', // https://www.algolia.com/doc/api-reference/api-parameters/snippetEllipsisText/ tagFilters: mapToAlgoliaFilters(tagsByFacet), // https://www.algolia.com/doc/api-reference/api-parameters/tagFilters/ }, }, ], transformResponse({ hits }) { // Re-order mixed hits so Docs stay in place while non-Doc items // are time-sorted, and compute a matchingHeading to deep-link // to the best title match. return hits.map((nestedHits) => { const docIndices = new Map(); nestedHits.forEach((item, index) => { if (item.type === 'Doc') { docIndices.set(index, item); } }); const nonDocItems = nestedHits .filter((item) => item.type !== 'Doc') .sort((a, b) => (b.unixTimestamp ?? -Infinity) - (a.unixTimestamp ?? -Infinity)); let nonDocIndex = 0; docIndices.forEach((item, index) => { if (item._highlightResult && item._highlightResult.titles) { const matchedIndex = item._highlightResult.titles.findIndex( (title) => title.t.matchLevel === 'full' ); if (matchedIndex !== -1) { const matchedTopLevelTitle = item.titles[matchedIndex]; if (matchedTopLevelTitle) { item.matchingHeading = `#${matchedTopLevelTitle.h}`; } } } nestedHits[index] = item; }); return nestedHits.map((item, index) => docIndices.has(index) ? docIndices.get(index) : nonDocItems[nonDocIndex++] ); }); }, }); }, templates: { noResults({ state, html }) { // Clear preview so we don’t show stale content for a query with no hits. state.context.preview = null; if (!state.query) return; return html`

No results for ${state.query}

Believe this query should return results? Let us know .

`; }, header({ items, html }) { if (!items.length) return; return html` Results

`; }, item({ item, components, html }) { const matchingHeading = item.matchingHeading || ''; // Helper: render “product” as deduped badges const renderProduct = (product) => { if (Array.isArray(product)) { const seen = new Set(); return product .filter(p => { const k = (p || '').toLowerCase(); if (seen.has(k)) return false; seen.add(k); return true; }) .map(p => html`

${p}

`); } else if (product) { return html`

${product}

`; } return null; }; // If previewable, navigate within the panel; otherwise open a new tab. const aTag = SUPPORTED_PREVIEW_TYPES.includes((item.type || '').toLowerCase()) ? html`

${components.Highlight({ hit: item, attribute: 'title' })}

${ item.text ? html`

${ item.text ? components.Snippet({ hit: item, attribute: 'text' }) : item.intro ? components.Snippet({ hit: item, attribute: 'intro' }) : item.description ? components.Snippet({ hit: item, attribute: 'description' }) : '' }

` : html`

${ item.intro ? components.Snippet({ hit: item, attribute: 'intro' }) : item.description ? components.Snippet({ hit: item, attribute: 'description' }) : '' }

` }

${item.type}

${renderProduct(item.product)} ${item.api ? html`

${item.api}

` : ''} ${item.version ? html`

${item.version}

` : ''}

` : html`

${components.Highlight({ hit: item, attribute: 'title' })}

${ item.breadcrumbs ? html`

` : '' }

${item.type}

${renderProduct(item.product)} ${item.version ? html`

${item.version}

` : ''}

`; return html`${aTag}`; }, }, getItemUrl({ item }) { return item.url || item.objectID; }, onActive({ item, setContext, state }) { const ctx = state?.context || {}; setContext({ ...ctx, preview: item }); }, } ]; }, }); /** * Expose the Autocomplete instance globally so other UI (our custom dropdown) * can drive its context without capturing stale references across re-renders. * This is intentionally namespaced and “private”. See notes below. */ window.__algoliaAutocompleteInstance = autocompleteInstance; } window.addEventListener('DOMContentLoaded', initAlgolia);

Simple online edits

Contribution guide