0
spark-submit 
--master yarn 
--deploy-mode cluster 
--num-executors=8 
--executor-memory=4G 
--driver-memory=10G 
--conf spark.yarn.submit.waitAppCompletion=false 
--conf spark.yarn.maxAppAttempts=1 
--conf spark.default.parallelism=16 
--jars jars/hive-warehouse-connector-assembly.jar 
--class classnameMain   jarname.jar TEST1
        
        
object KC {

def sendMsg(topic: String, key: String, value: String): Unit = {
    val kafkaServer = configManager.getString("Kafka.Server")
    val props = new Properties()
    props.put("bootstrap.servers", kafkaServer)
    props.put("key.serializer", )
    props.put("value.serializer",)
    props.put("batch.size", "1")
    props.put("acks", "all")
    val producer = new KafkaProducer[String, String](props)
    val message = new ProducerRecord[String, String](topic, key, value)
    producer.send(message)
   }
    
def completeMSG(Id: BigInt, tableName: String, topic: String, envi:String ): Unit = {
        val kafkaServer = configManager.getString("Kafka.Server")
        val props = new Properties()
        props.put("bootstrap.servers", kafkaServer)
        props.put("key.serializer", )
        props.put("value.serializer",)
        props.put("batch.size", "1")
        props.put("acks", "all")
        val producer = new KafkaProducer[String, String](props)
        val key = key_names
        val value = value_names
        val message = new ProducerRecord[String, String](s"{$envi_$topic}", key, value)
        producer.send(message)
    }
}
          
        
 def main(args: Array[String]): Unit = {
     configManager.setup("application.conf")
     val warehouseLocation = ""
     val spark = SparkSession.builder()
       .master("local[*]")
       .appName("Spark Hive Example")
       .config("spark.sql.warehouse.dir", warehouseLocation)
       .config("hive.metastore.uris", "")
       .config("hive.exec.dynamic.partition", "")
       .config("hive.exec.dynamic.partition.mode", "")
       .enableHiveSupport()
       .getOrCreate()
        
        val Id = 12345
        val tableName = employee
        logManager.logInfo(spark, jobId, "start of Kafka message")
        val topic = "XLEMP"
        val key = "XL"
        val value = "Id +valueId+"
        sendMsg(topic, key, value)
        logManager.logInfo(spark, jobId, "end of Kafka message")
   }
        
import KC
    
employeedetail.scala --> this is a file under which we are calling completeMSG method

KC.completeMSG(Id, "tblename", topicname, envi = ?)

So I want (TEST1) which is envi in completeMSG method. TEST1 is a argument which nothing but environment name. It may be TEST2 , Q2 and so on. I want to pass this argument(environment) name along with kafka topic. So it should be like envi_topic how the value of envi comes ?

dataeng
  • 11
  • 4
  • Show your main method, and where completeMSG is called, please... Otherwise, you've basically asked the same question about 4 different ways – OneCricketeer Jun 29 '22 at 14:51
  • Also,it should be `s"${envi}_${topic}"`, and youre not using Spark to actually produce to Kafka, only Scala – OneCricketeer Jun 29 '22 at 14:54
  • https://stackoverflow.com/users/2308683/onecricketeer : Actually, it is not. The previous question also did not resolve the problem. That solution was partially resolve the issues. So I asked the question this time in more descriptive way. – dataeng Jun 29 '22 at 15:16
  • What exactly is the problem? I've shown you how to do string interpolation 3 times now. Please see example - https://ideone.com/qCCcMQ I am asking for you to show the main method because that is where `TEST1` string enters the application. From there, you need to show where `completeMSG` is used and gets its parameters from as part of a [mcve] – OneCricketeer Jun 29 '22 at 15:30
  • https://stackoverflow.com/users/2308683/onecricketeer : It is not related to string interpolation. We follow another approach as I have to deliver the things. I am going to modify my question again along with Main method. So that you will get the clear idea – dataeng Jun 29 '22 at 17:42
  • https://stackoverflow.com/users/2308683/onecricketeer : I modifed my question, so might be this time you will get the idea, what is my requirement exactly. – dataeng Jun 30 '22 at 06:15
  • 1) Use `@` to tag people, not links 2) You're never using `args(0)` as I answered in the last question... And I have so many other questions - Where exactly are you calling `KC.completeMSG` from the main method? What is `sendMsg` function? Why is your topic string empty there? Why are key and value strings, and also empty? What's the point of the `Id` if you never use it? What is `tblname` ? You do understand what "reproducible" and "minimal" part of [mcve] mean, yes? – OneCricketeer Jun 30 '22 at 14:03
  • @OneCricketeer : I have made the changes in code, I added sendMsg method, added tblname, key, value and topic name. regarding your args(0) value , we want to pass that argument on console. we can also added in main method. But I don't understand how to call that args(0) in code and how to get that value in in ENVI vaiable – dataeng Jul 01 '22 at 08:01

0 Answers0