Finds all the files matching a specified pattern in Spark.

ListFiles.scala

import org.apache.hadoop.fs.{FileStatus, Path}
import org.apache.spark.sql.SparkSession

object ListFiles extends App {
  val spark = SparkSession.builder.config("spark.master", "local[1]").getOrCreate()

  val path = new Path("/data/wiki-dumps/dumps/*wiki-*-pages-articles-multistream.xml*")
  val fileSystem = path.getFileSystem(spark.sparkContext.hadoopConfiguration)
  val files: List[FileStatus] = fileSystem.globStatus(path).toList

  files.foreach(println)
}