Better timestamp support with timestamp[us] instead of timestamp[ns] (~290 000 years vs. 585 years respectively)

dbascoules · dbascoules · commit b3a47b8d2c3e · 2025-12-15T09:39:31.000+01:00
timestamp_nanos_opt() support only dates between 1677 and 2262
diff --git a/connectorx/src/destinations/arrow/arrow_assoc.rs b/connectorx/src/destinations/arrow/arrow_assoc.rs
@@ -288,9 +288,7 @@ fn naive_date_to_arrow(nd: NaiveDate) -> i32 {
 }
 
 fn naive_datetime_to_arrow(nd: NaiveDateTime) -> i64 {
-    nd.and_utc()
-        .timestamp_nanos_opt()
-        .unwrap_or_else(|| panic!("out of range DateTime"))
+    nd.and_utc().timestamp_micros()
 }
 
 impl ArrowAssoc for Option<NaiveDate> {
@@ -328,10 +326,10 @@ impl ArrowAssoc for NaiveDate {
 }
 
 impl ArrowAssoc for Option<NaiveDateTime> {
-    type Builder = TimestampNanosecondBuilder;
+    type Builder = TimestampMicrosecondBuilder;
 
     fn builder(nrows: usize) -> Self::Builder {
-        TimestampNanosecondBuilder::with_capacity(nrows)
+        TimestampMicrosecondBuilder::with_capacity(nrows)
     }
 
     fn append(builder: &mut Self::Builder, value: Option<NaiveDateTime>) -> Result<()> {
@@ -342,17 +340,17 @@ impl ArrowAssoc for Option<NaiveDateTime> {
     fn field(header: &str) -> Field {
         Field::new(
             header,
-            ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),
+            ArrowDataType::Timestamp(TimeUnit::Microsecond, None),
             true,
         )
     }
 }
 
 impl ArrowAssoc for NaiveDateTime {
-    type Builder = TimestampNanosecondBuilder;
+    type Builder = TimestampMicrosecondBuilder;
 
     fn builder(nrows: usize) -> Self::Builder {
-        TimestampNanosecondBuilder::with_capacity(nrows)
+        TimestampMicrosecondBuilder::with_capacity(nrows)
     }
 
     fn append(builder: &mut Self::Builder, value: NaiveDateTime) -> Result<()> {
@@ -363,7 +361,7 @@ impl ArrowAssoc for NaiveDateTime {
     fn field(header: &str) -> Field {
         Field::new(
             header,
-            ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),
+            ArrowDataType::Timestamp(TimeUnit::Microsecond, None),
             false,
         )
     }
diff --git a/connectorx/src/destinations/arrowstream/arrow_assoc.rs b/connectorx/src/destinations/arrowstream/arrow_assoc.rs
@@ -285,9 +285,7 @@ fn naive_date_to_arrow(nd: NaiveDate) -> i32 {
 }
 
 fn naive_datetime_to_arrow(nd: NaiveDateTime) -> i64 {
-    nd.and_utc()
-        .timestamp_nanos_opt()
-        .unwrap_or_else(|| panic!("out of range DateTime"))
+    nd.and_utc().timestamp_micros()
 }
 
 impl ArrowAssoc for Option<NaiveDate> {
@@ -325,10 +323,10 @@ impl ArrowAssoc for NaiveDate {
 }
 
 impl ArrowAssoc for Option<NaiveDateTime> {
-    type Builder = TimestampNanosecondBuilder;
+    type Builder = TimestampMicrosecondBuilder;
 
     fn builder(nrows: usize) -> Self::Builder {
-        TimestampNanosecondBuilder::with_capacity(nrows)
+        TimestampMicrosecondBuilder::with_capacity(nrows)
     }
 
     fn append(builder: &mut Self::Builder, value: Option<NaiveDateTime>) -> Result<()> {
@@ -339,17 +337,17 @@ impl ArrowAssoc for Option<NaiveDateTime> {
     fn field(header: &str) -> Field {
         Field::new(
             header,
-            ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),
+            ArrowDataType::Timestamp(TimeUnit::Microsecond, None),
             true,
         )
     }
 }
 
 impl ArrowAssoc for NaiveDateTime {
-    type Builder = TimestampNanosecondBuilder;
+    type Builder = TimestampMicrosecondBuilder;
 
     fn builder(nrows: usize) -> Self::Builder {
-        TimestampNanosecondBuilder::with_capacity(nrows)
+        TimestampMicrosecondBuilder::with_capacity(nrows)
     }
 
     fn append(builder: &mut Self::Builder, value: NaiveDateTime) -> Result<()> {
@@ -360,7 +358,7 @@ impl ArrowAssoc for NaiveDateTime {
     fn field(header: &str) -> Field {
         Field::new(
             header,
-            ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),
+            ArrowDataType::Timestamp(TimeUnit::Microsecond, None),
             false,
         )
     }

Original file line number	Diff line number	Diff line change
`@@ -288,9 +288,7 @@ fn naive_date_to_arrow(nd: NaiveDate) -> i32 {`
`288`	`288`	`}`
`289`	`289`
`290`	`290`	`fn naive_datetime_to_arrow(nd: NaiveDateTime) -> i64 {`
`291`		`- nd.and_utc()`
`292`		`- .timestamp_nanos_opt()`
`293`		`- .unwrap_or_else(\|\| panic!("out of range DateTime"))`
	`291`	`+ nd.and_utc().timestamp_micros()`
`294`	`292`	`}`
`295`	`293`
`296`	`294`	`impl ArrowAssoc for Option<NaiveDate> {`
`@@ -328,10 +326,10 @@ impl ArrowAssoc for NaiveDate {`
`328`	`326`	`}`
`329`	`327`
`330`	`328`	`impl ArrowAssoc for Option<NaiveDateTime> {`
`331`		`- type Builder = TimestampNanosecondBuilder;`
	`329`	`+ type Builder = TimestampMicrosecondBuilder;`
`332`	`330`
`333`	`331`	`fn builder(nrows: usize) -> Self::Builder {`
`334`		`- TimestampNanosecondBuilder::with_capacity(nrows)`
	`332`	`+ TimestampMicrosecondBuilder::with_capacity(nrows)`
`335`	`333`	`}`
`336`	`334`
`337`	`335`	`fn append(builder: &mut Self::Builder, value: Option<NaiveDateTime>) -> Result<()> {`
`@@ -342,17 +340,17 @@ impl ArrowAssoc for Option<NaiveDateTime> {`
`342`	`340`	`fn field(header: &str) -> Field {`
`343`	`341`	`Field::new(`
`344`	`342`	`header,`
`345`		`- ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),`
	`343`	`+ ArrowDataType::Timestamp(TimeUnit::Microsecond, None),`
`346`	`344`	`true,`
`347`	`345`	`)`
`348`	`346`	`}`
`349`	`347`	`}`
`350`	`348`
`351`	`349`	`impl ArrowAssoc for NaiveDateTime {`
`352`		`- type Builder = TimestampNanosecondBuilder;`
	`350`	`+ type Builder = TimestampMicrosecondBuilder;`
`353`	`351`
`354`	`352`	`fn builder(nrows: usize) -> Self::Builder {`
`355`		`- TimestampNanosecondBuilder::with_capacity(nrows)`
	`353`	`+ TimestampMicrosecondBuilder::with_capacity(nrows)`
`356`	`354`	`}`
`357`	`355`
`358`	`356`	`fn append(builder: &mut Self::Builder, value: NaiveDateTime) -> Result<()> {`
`@@ -363,7 +361,7 @@ impl ArrowAssoc for NaiveDateTime {`
`363`	`361`	`fn field(header: &str) -> Field {`
`364`	`362`	`Field::new(`
`365`	`363`	`header,`
`366`		`- ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),`
	`364`	`+ ArrowDataType::Timestamp(TimeUnit::Microsecond, None),`
`367`	`365`	`false,`
`368`	`366`	`)`
`369`	`367`	`}`
Original file line number	Diff line number	Diff line change
`@@ -285,9 +285,7 @@ fn naive_date_to_arrow(nd: NaiveDate) -> i32 {`
`285`	`285`	`}`
`286`	`286`
`287`	`287`	`fn naive_datetime_to_arrow(nd: NaiveDateTime) -> i64 {`
`288`		`- nd.and_utc()`
`289`		`- .timestamp_nanos_opt()`
`290`		`- .unwrap_or_else(\|\| panic!("out of range DateTime"))`
	`288`	`+ nd.and_utc().timestamp_micros()`
`291`	`289`	`}`
`292`	`290`
`293`	`291`	`impl ArrowAssoc for Option<NaiveDate> {`
`@@ -325,10 +323,10 @@ impl ArrowAssoc for NaiveDate {`
`325`	`323`	`}`
`326`	`324`
`327`	`325`	`impl ArrowAssoc for Option<NaiveDateTime> {`
`328`		`- type Builder = TimestampNanosecondBuilder;`
	`326`	`+ type Builder = TimestampMicrosecondBuilder;`
`329`	`327`
`330`	`328`	`fn builder(nrows: usize) -> Self::Builder {`
`331`		`- TimestampNanosecondBuilder::with_capacity(nrows)`
	`329`	`+ TimestampMicrosecondBuilder::with_capacity(nrows)`
`332`	`330`	`}`
`333`	`331`
`334`	`332`	`fn append(builder: &mut Self::Builder, value: Option<NaiveDateTime>) -> Result<()> {`
`@@ -339,17 +337,17 @@ impl ArrowAssoc for Option<NaiveDateTime> {`
`339`	`337`	`fn field(header: &str) -> Field {`
`340`	`338`	`Field::new(`
`341`	`339`	`header,`
`342`		`- ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),`
	`340`	`+ ArrowDataType::Timestamp(TimeUnit::Microsecond, None),`
`343`	`341`	`true,`
`344`	`342`	`)`
`345`	`343`	`}`
`346`	`344`	`}`
`347`	`345`
`348`	`346`	`impl ArrowAssoc for NaiveDateTime {`
`349`		`- type Builder = TimestampNanosecondBuilder;`
	`347`	`+ type Builder = TimestampMicrosecondBuilder;`
`350`	`348`
`351`	`349`	`fn builder(nrows: usize) -> Self::Builder {`
`352`		`- TimestampNanosecondBuilder::with_capacity(nrows)`
	`350`	`+ TimestampMicrosecondBuilder::with_capacity(nrows)`
`353`	`351`	`}`
`354`	`352`
`355`	`353`	`fn append(builder: &mut Self::Builder, value: NaiveDateTime) -> Result<()> {`
`@@ -360,7 +358,7 @@ impl ArrowAssoc for NaiveDateTime {`
`360`	`358`	`fn field(header: &str) -> Field {`
`361`	`359`	`Field::new(`
`362`	`360`	`header,`
`363`		`- ArrowDataType::Timestamp(TimeUnit::Nanosecond, None),`
	`361`	`+ ArrowDataType::Timestamp(TimeUnit::Microsecond, None),`
`364`	`362`	`false,`
`365`	`363`	`)`
`366`	`364`	`}`