Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -267,4 +267,5 @@ _includes/code/csharp/bin
_includes/code/csharp/obj
*.sln

# Exclude WCD backups
tests/backups/
183 changes: 183 additions & 0 deletions _includes/code/automated-backup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
import weaviate
from weaviate.classes.init import Auth
from weaviate.classes.data import GeoCoordinate
import json
import os
from datetime import datetime

# Custom JSON encoder to handle datetime and Weaviate-specific objects
class WeaviateEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime):
return obj.isoformat()
if isinstance(obj, GeoCoordinate):
return {
"latitude": obj.latitude,
"longitude": obj.longitude
}
# Handle any other non-serializable objects by converting to string
try:
return super().default(obj)
except TypeError:
return str(obj)

# Configuration
wcd_url = os.environ["WEAVIATE_URL"]
wcd_api_key = os.environ["WEAVIATE_API_KEY"]
BASE_DIR = "/Users/ivandespot/dev/docs/tests/backups"
BACKUP_DIR = os.path.join(BASE_DIR, f"backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}")

# Create backup directory
os.makedirs(BACKUP_DIR, exist_ok=True)

# Connect to Weaviate Cloud
client = weaviate.connect_to_weaviate_cloud(
cluster_url=wcd_url, auth_credentials=Auth.api_key(wcd_api_key)
)

try:
# Get all collections
collections = client.collections.list_all()
print(f"Found {len(collections)} collections to back up")

backup_metadata = {
"timestamp": datetime.now().isoformat(),
"cluster_url": wcd_url,
"collections": [],
}

# Back up each collection
for collection_name in collections:
print(f"\nBacking up collection: {collection_name}")
collection = client.collections.get(collection_name)

# Get collection config (schema)
config = collection.config.get()
config_dict = {
"name": collection_name,
"description": config.description,
"properties": [
{
"name": prop.name,
"data_type": prop.data_type.value,
"description": prop.description,
}
for prop in config.properties
],
"vectorizer_config": str(config.vectorizer_config),
"vector_index_config": str(config.vector_index_config),
"generative_config": (
str(config.generative_config) if config.generative_config else None
),
"replication_config": (
str(config.replication_config) if config.replication_config else None
),
"multi_tenancy_config": (
str(config.multi_tenancy_config)
if config.multi_tenancy_config
else None
),
}

# Check if multi-tenancy is enabled
is_multi_tenant = config.multi_tenancy_config and config.multi_tenancy_config.enabled

# Save collection config
config_file = os.path.join(BACKUP_DIR, f"{collection_name}_config.json")
with open(config_file, "w") as f:
json.dump(config_dict, f, indent=2, cls=WeaviateEncoder)

collection_metadata = {
"name": collection_name,
"config_file": f"{collection_name}_config.json",
"is_multi_tenant": is_multi_tenant,
"tenants": []
}

if is_multi_tenant:
# Get all tenants (returns list of tenant names as strings)
tenants = collection.tenants.get()
print(f" Found {len(tenants)} tenants")

# Back up each tenant
for tenant_name in tenants:
print(f" Backing up tenant: {tenant_name}")

# Get tenant-specific collection
tenant_collection = collection.with_tenant(tenant_name)

# Export tenant objects
objects = []
object_count = 0

try:
for item in tenant_collection.iterator(include_vector=True):
obj = {
"uuid": str(item.uuid),
"properties": item.properties,
"vector": item.vector,
}
objects.append(obj)
object_count += 1

if object_count % 1000 == 0:
print(f" Exported {object_count} objects...")

# Save tenant objects
objects_file = os.path.join(BACKUP_DIR, f"{collection_name}_{tenant_name}_objects.json")
with open(objects_file, "w") as f:
json.dump(objects, f, indent=2, cls=WeaviateEncoder)

print(f" ✓ Backed up {object_count} objects for tenant {tenant_name}")

collection_metadata["tenants"].append({
"tenant_name": tenant_name,
"object_count": object_count,
"objects_file": f"{collection_name}_{tenant_name}_objects.json"
})
except Exception as e:
print(f" ⚠ Warning: Could not back up tenant {tenant_name}: {e}")
collection_metadata["tenants"].append({
"tenant_name": tenant_name,
"object_count": 0,
"error": str(e)
})
else:
# Non-multi-tenant collection - backup normally
objects = []
object_count = 0

for item in collection.iterator(include_vector=True):
obj = {
"uuid": str(item.uuid),
"properties": item.properties,
"vector": item.vector,
}
objects.append(obj)
object_count += 1

if object_count % 1000 == 0:
print(f" Exported {object_count} objects...")

# Save objects
objects_file = os.path.join(BACKUP_DIR, f"{collection_name}_objects.json")
with open(objects_file, "w") as f:
json.dump(objects, f, indent=2, cls=WeaviateEncoder)

print(f" ✓ Backed up {object_count} objects")

collection_metadata["object_count"] = object_count
collection_metadata["objects_file"] = f"{collection_name}_objects.json"

backup_metadata["collections"].append(collection_metadata)

# Save backup metadata
metadata_file = os.path.join(BACKUP_DIR, "backup_metadata.json")
with open(metadata_file, "w") as f:
json.dump(backup_metadata, f, indent=2, cls=WeaviateEncoder)

print(f"\n✓ Backup completed successfully!")
print(f"Backup location: {BACKUP_DIR}")

finally:
client.close()
4 changes: 4 additions & 0 deletions _includes/code/csharp/AssemblyInfo.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
using Xunit;

// This forces all tests in this assembly to run sequentially
[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)]
158 changes: 158 additions & 0 deletions _includes/code/csharp/BackupsTest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
using Xunit;
using Weaviate.Client;
using Weaviate.Client.Models;
using System;
using System.Threading.Tasks;
using System.Threading;

// Run sequentially to prevent backup conflicts on the filesystem backend
[Collection("Sequential")]
public class BackupsTest : IAsyncLifetime
{
private WeaviateClient client;
private readonly BackupBackend _backend = new FilesystemBackend();

public async Task InitializeAsync()
{
client = await Connect.Local(
restPort: 8580,
grpcPort: 50551,
credentials: "root-user-key"
);

// Ensure a clean state
await CleanupCollections();
}

public Task DisposeAsync()
{
// The C# client manages connections automatically.
return Task.CompletedTask;
}

// Helper method to set up collections for tests
private async Task SetupCollections()
{
await CleanupCollections();

await client.Collections.Create(new CollectionConfig
{
Name = "Article",
Properties = [Property.Text("title")]
});

await client.Collections.Create(new CollectionConfig
{
Name = "Publication",
Properties = [Property.Text("title")]
});

await client.Collections.Use("Article").Data.Insert(new { title = "Dummy" });
await client.Collections.Use("Publication").Data.Insert(new { title = "Dummy" });
}

private async Task CleanupCollections()
{
if (await client.Collections.Exists("Article")) await client.Collections.Delete("Article");
if (await client.Collections.Exists("Publication")) await client.Collections.Delete("Publication");
}

[Fact]
public async Task TestBackupAndRestoreLifecycle()
{
await SetupCollections();
string backupId = "my-very-first-backup";

// START CreateBackup
var createResult = await client.Backups.CreateSync(
new BackupCreateRequest(
Id: backupId,
Backend: _backend,
Include: ["Article", "Publication"]
)
);

Console.WriteLine($"Status: {createResult.Status}");
// END CreateBackup

Assert.Equal(BackupStatus.Success, createResult.Status);

// START StatusCreateBackup
var createStatus = await client.Backups.GetStatus(_backend, backupId);

Console.WriteLine($"Backup ID: {createStatus.Id}, Status: {createStatus.Status}");
// END StatusCreateBackup

Assert.Equal(BackupStatus.Success, createStatus.Status);

// Delete all classes before restoring
await client.Collections.DeleteAll();
Assert.False(await client.Collections.Exists("Article"));
Assert.False(await client.Collections.Exists("Publication"));

// START RestoreBackup
var restoreResult = await client.Backups.RestoreSync(
new BackupRestoreRequest(
Id: backupId,
Backend: _backend,
Exclude: ["Article"] // Exclude Article from restoration
)
);

Console.WriteLine($"Restore Status: {restoreResult.Status}");
// END RestoreBackup

Assert.Equal(BackupStatus.Success, restoreResult.Status);

// Verify that Publication was restored and Article was excluded
Assert.True(await client.Collections.Exists("Publication"));
Assert.False(await client.Collections.Exists("Article"));

// START StatusRestoreBackup
// Note: In C#, restore status is often tracked via the returned operation or by polling if async.
// GetRestoreStatus checks the status of a specific restore job.
// Since we ran RestoreSync, we know it is done.
// We can inspect the result returned from RestoreSync directly.
Console.WriteLine($"Restore ID: {restoreResult.Id}, Status: {restoreResult.Status}");
// END StatusRestoreBackup

Assert.Equal(BackupStatus.Success, restoreResult.Status);

// Clean up
await client.Collections.Delete("Publication");
}

[Fact]
public async Task TestCancelBackup()
{
await SetupCollections();
string backupId = "some-unwanted-backup";

// Start a backup to cancel (Async, creates the operation but returns immediately)
CancellationToken cancellationToken = new CancellationToken();
var backupOperation = await client.Backups.Create(
new BackupCreateRequest(
Id: backupId,
Backend: _backend,
Include: ["Article", "Publication"]
),
cancellationToken
);

Console.WriteLine($"Backup started with ID: {backupOperation.Current.Id}");

// START CancelBackup
await backupOperation.Cancel(cancellationToken);
// END CancelBackup

// Wait for the cancellation to be processed
var finalStatus = await backupOperation.WaitForCompletion();

// Verify status
Assert.Equal(BackupStatus.Canceled, finalStatus.Status);

// Clean up
await client.Collections.Delete("Article");
await client.Collections.Delete("Publication");
}
}
13 changes: 4 additions & 9 deletions _includes/code/csharp/ConfigureBQTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,8 @@ public class ConfigureBQTest : IAsyncLifetime
public async Task InitializeAsync()
{
// START ConnectCode
// Note: The C# client doesn't support setting headers like 'X-OpenAI-Api-Key' via the constructor for local connections.
// This must be configured in Weaviate's environment variables.
client = new WeaviateClient(new ClientConfiguration { RestAddress = "localhost", RestPort = 8080 });
client = await Connect.Local();
// END ConnectCode

// Clean slate for each test
if (await client.Collections.Exists(COLLECTION_NAME))
{
await client.Collections.Delete(COLLECTION_NAME);
}
}

// Runs after each test
Expand All @@ -37,6 +29,7 @@ public Task DisposeAsync()
[Fact]
public async Task TestEnableBQ()
{
await client.Collections.Delete(COLLECTION_NAME);
// START EnableBQ
await client.Collections.Create(new CollectionConfig
{
Expand All @@ -59,6 +52,7 @@ await client.Collections.Create(new CollectionConfig
[Fact]
public async Task TestUpdateSchema()
{
await client.Collections.Delete(COLLECTION_NAME);
// Note: Updating quantization settings on an existing collection is not supported by Weaviate
// and will result in an error, as noted in the Java test. This test demonstrates the syntax for attempting the update.
var collection = await client.Collections.Create(new CollectionConfig
Expand All @@ -80,6 +74,7 @@ await collection.Config.Update(c =>
[Fact]
public async Task TestBQWithOptions()
{
await client.Collections.Delete(COLLECTION_NAME);
// START BQWithOptions
await client.Collections.Create(new CollectionConfig
{
Expand Down
Loading
Loading