Mainframe Connector: user-specified charset (#901)

* Mainframe Connector: increment version * Mainframe Connector: accept user-specified charset from envvar and command-line arg
GoogleCloudPlatform · Oct 7, 2022 · 2c2d13e · 2c2d13e
1 parent 30fdbf3
commit 2c2d13e
Show file tree

Hide file tree

Showing 15 changed files with 60 additions and 10 deletions.
diff --git a/tools/bigquery-zos-mainframe-connector/gszutil/build.sbt b/tools/bigquery-zos-mainframe-connector/gszutil/build.sbt
@@ -16,7 +16,7 @@
  */
 organization := "com.google.cloud.imf"
 name := "mainframe-connector"
-version := "5.7.1"
+version := "5.7.2"
 
 scalaVersion := "2.13.8"
 

diff --git a/...y-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/ExportConfig.scala b/...y-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/ExportConfig.scala
@@ -27,6 +27,7 @@ case class ExportConfig(
                          timeoutMinutes: Int = 90,
                          vartext: Boolean = false,
                          runMode: String = "parallel",
+                         encoding: Option[String] = None,
                          picTCharset: Option[String] = None,
 
                          bucket: String = "",

diff --git a/...mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/ExportOptionParser.scala b/...mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/ExportOptionParser.scala
@@ -162,6 +162,11 @@ object ExportOptionParser
     .action((x,c) => c.copy(sync = x))
 
   // Custom Options
+  opt[String]("encoding")
+    .optional()
+    .text("(optional) charset used for encoding and decoding character fields. Overrides default set by ENCODING environment variable.")
+    .action((x, c) => c.copy(encoding = Option(x)))
+
   opt[String]("stats_table")
     .optional()
     .text("tablespec of table to insert stats")

diff --git a/...y-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsUtilConfig.scala b/...y-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsUtilConfig.scala
@@ -91,6 +91,7 @@ case class GsUtilConfig(source: String = "INFILE",
                         // Custom
                         schemaProvider: Option[SchemaProvider] = None,
                         picTCharset: Option[String] = None,
+                        encoding: Option[String] = None,
                         statsTable: String = "",
                         remote: Boolean = false,
                         remoteHost: String = "",

diff --git a/...mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsUtilOptionParser.scala b/...mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsUtilOptionParser.scala
@@ -110,6 +110,10 @@ object GsUtilOptionParser extends OptionParser[GsUtilConfig]("gsutil") with ArgP
         .action { (x, c) =>
           c.copy(tfGCS = x)
         },
+      opt[String]("encoding")
+        .optional()
+        .text("(optional) charset used for encoding and decoding character fields. Overrides default set by ENCODING environment variable.")
+        .action((x,c) => c.copy(encoding = Option(x))),
       opt[String]("pic_t_charset")
         .optional()
         .text("(optional) charset used for encoding and decoding international strings, used with PIC T copybook type, default is EBCDIC")

diff --git a/...-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsZUtilConfig.scala b/...-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsZUtilConfig.scala
@@ -27,6 +27,7 @@ case class GsZUtilConfig(
                           projectId: String = "",
                           datasetId: String = "",
                           location: String = "US",
+                          encoding: Option[String] = None,
                           picTCharset: Option[String] = None,
                           timeOutMinutes: Option[Int] = None,
                           keepAliveTimeInSeconds: Option[Int] = None)
diff --git a/...ainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsZUtilOptionParser.scala b/...ainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/GsZUtilOptionParser.scala
@@ -59,6 +59,11 @@ with ArgParser[GsZUtilConfig]{
     .text("port of GRecv transcoding service (default: 52701 or SRVPORT environment variable)")
     .action((x,c) => c.copy(remoteHost = x))
 
+  opt[String]("encoding")
+    .optional()
+    .text("(optional) charset used for encoding and decoding character fields. Overrides default set by ENCODING environment variable.")
+    .action((x, c) => c.copy(encoding = Option(x)))
+
   opt[String]("pic_t_charset")
     .optional()
     .text("(optional) charset used for encoding and decoding international strings, used with PIC T copybook type, default is EBCDIC")

diff --git a/...igquery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/cmd/Cp.scala b/...igquery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/cmd/Cp.scala
@@ -54,7 +54,7 @@ object Cp extends Command[GsUtilConfig] with Logging {
       case Some(x) => {
         logger.info("Merging copybook with provided transformations ...")
 
-        val sch = c.schemaProvider.getOrElse(zos.loadCopyBook(c.copyBook, c.picTCharset))
+        val sch = c.schemaProvider.getOrElse(zos.loadCopyBook(c.copyBook, c.encoding, c.picTCharset))
         logger.info(s"Current Schema: ${sch.toString}")
 
         val newSchema = merge(sch, x)
@@ -63,7 +63,7 @@ object Cp extends Command[GsUtilConfig] with Logging {
       }
       case None => {
         logger.info("Use original copybook")
-        c.schemaProvider.getOrElse(zos.loadCopyBook(c.copyBook, c.picTCharset))
+        c.schemaProvider.getOrElse(zos.loadCopyBook(c.copyBook, c.encoding, c.picTCharset))
       }
     }
     val in: ZRecordReaderT = c.testInput.getOrElse(zos.readCloudDD(c.source))

diff --git a/...ery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/cmd/Export.scala b/...ery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/cmd/Export.scala
@@ -63,7 +63,7 @@ object Export extends Command[ExportConfig] with Logging {
         CopyBook(zos.readDSNLines(MVSStorage.parseDSN(cfg.cobDsn)).mkString("\n"), picTCharset = cfg.picTCharset)
       } else {
         logger.info(s"reading copybook from DD:COPYBOOK")
-        zos.loadCopyBook("COPYBOOK",  cfg.picTCharset)
+        zos.loadCopyBook("COPYBOOK", cfg.encoding,  cfg.picTCharset)
       }
 
     try {

diff --git a/...ry-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/cmd/GsZUtil.scala b/...ry-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/bqsh/cmd/GsZUtil.scala
@@ -56,7 +56,7 @@ object GsZUtil extends Command[GsZUtilConfig] with Logging {
         CopyBook(zos.readDSNLines(MVSStorage.parseDSN(c.cobDsn)).mkString("\n"), picTCharset = c.picTCharset)
       } else {
         logger.info(s"reading copybook from DD:COPYBOOK")
-        zos.loadCopyBook("COPYBOOK", c.picTCharset)
+        zos.loadCopyBook("COPYBOOK", c.encoding, c.picTCharset)
       }
     //TODO read FLDINFO DD and merge field info
 

diff --git a/...nframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/CharsetTranscoder.scala b/...nframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/CharsetTranscoder.scala
@@ -0,0 +1,17 @@
+package com.google.cloud.imf.gzos
+
+import com.google.cloud.gszutil.Transcoder
+
+import java.nio.charset.Charset
+
+/** Transcoder with user-specified character set
+  * @param encoding name of character set
+  */
+case class CharsetTranscoder(encoding: String) extends Transcoder {
+  override val charset: Charset = Charset.forName(encoding)
+  override val SP: Byte = {
+    val bytes = charset.encode(" ").array()
+    require(bytes.length == 1, "multi-byte space character not supported")
+    bytes.head
+  }
+}
diff --git a/...ery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/Ebcdic.scala b/...ery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/Ebcdic.scala
@@ -20,7 +20,19 @@ import java.nio.charset.Charset
 
 import com.google.cloud.gszutil.Transcoder
 
+/** Default Transcoder
+  * Uses EBCDIC US charset by default.
+  * Obtains charset name from ENCODING environment variable.
+  */
 case object Ebcdic extends Transcoder {
-  override final val charset: Charset = new EBCDIC1()
+  override final val charset: Charset = {
+    sys.env.get("ENCODING") match {
+      case Some(charset) =>
+        System.out.println(s"Using Charset '$charset'")
+        Charset.forName(charset)
+      case None =>
+        new EBCDIC1()
+    }
+  }
   override val SP: Byte = 0x40
 }
diff --git a/...gquery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/IBM.scala b/...gquery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/IBM.scala
@@ -121,11 +121,15 @@ object IBM extends MVS with Logging {
     }
   }
 
-  override def loadCopyBook(dd: String, picTCharset: Option[String] = None) : CopyBook = {
+  override def loadCopyBook(dd: String, encoding: Option[String] = None, picTCharset: Option[String] = None) : CopyBook = {
     val raw = readDDString(dd, "\n")
     logger.info(s"Parsing copy book:\n$raw")
     try {
-      val copyBook = CopyBook(raw, Ebcdic, picTCharset = picTCharset)
+      val transcoder = encoding match {
+        case Some(charsetName) => CharsetTranscoder(charsetName)
+        case None => Ebcdic
+      }
+      val copyBook = CopyBook(raw, transcoder, picTCharset = picTCharset)
       logger.info(s"Loaded copy book with LRECL=${copyBook.LRECL}")
       copyBook
     } catch {

diff --git a/...uery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/Linux.scala b/...uery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/Linux.scala
@@ -96,7 +96,7 @@ object Linux extends MVS with Logging {
 
   override def readKeyfile(): Array[Byte] = Array.empty
 
-  override def loadCopyBook(dd: String, picTCharset: Option[String] = Some("UTF-8")): CopyBook = {
+  override def loadCopyBook(dd: String, encoding: Option[String] = None, picTCharset: Option[String] = Some("UTF-8")): CopyBook = {
     val ddValue = System.getenv(dd)
     require(ddValue != null, s"$dd environment variable not defined")
     val ddPath = Paths.get(ddValue)

diff --git a/...gquery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/MVS.scala b/...gquery-zos-mainframe-connector/gszutil/src/main/scala/com/google/cloud/imf/gzos/MVS.scala
@@ -40,7 +40,7 @@ trait MVS {
   def getCredentialProvider(): CredentialProvider
   def readKeyfile(): Array[Byte]
   def listPDS(dsn: DSN): Iterator[PDSMemberInfo]
-  def loadCopyBook(dd: String, picTCharset: Option[String] = None): CopyBook
+  def loadCopyBook(dd: String, encoding: Option[String] = None, picTCharset: Option[String] = None): CopyBook
   def jobName: String
   def jobDate: String
   def jobTime: String