Re-setup docs generation from code (#202)

* Update packages, configurations etc for documentation gen

* Update documentation for jsdoc

* Add docs generation to semaphore pipeline

* Fix merge conflict related doc change

Author: milindl

.npmignore

@@ -16,4 +16,5 @@ deps/librdkafka/config.h
 /e2e
 /bench
 /ci
-/proto
+/proto
+/docs

.semaphore/semaphore.yml

@@ -87,7 +87,7 @@ blocks:
             - npm run install-from-source
             - make test
 
-  - name: "Linux amd64: Build, test, lint"
+  - name: "Linux amd64: Build, test, lint, docs"
     dependencies: [ ]
     task:
       agent:
@@ -110,6 +110,9 @@ blocks:
         - name: "ESLint"
           commands:
             - npx eslint lib/kafkajs
+        - name: "Docs"
+          commands:
+            - make docs
 
   - name: "Linux amd64: Performance"
     dependencies: [ ]

Makefile

@@ -74,14 +74,7 @@ define release
 endef
 
 docs: node_modules/.dirstamp
-	@rm -rf docs
-	@./node_modules/jsdoc/jsdoc.js --debug --destination docs \
-		--recurse -R ./README.md \
-		-c ./jsdoc.conf \
-		--tutorials examples/ ./lib
-
-gh-pages: node_modules/.dirstamp
-	@./make_docs.sh
+	@./util/generate-docs.sh
 
 release-patch:
 	@$(call release,patch)

jsdoc.conf

@@ -1,16 +1,23 @@
 {
-    "plugins": [],
+    "plugins": [
+        "plugins/markdown"
+    ],
     "recurseDepth": 10,
     "source": {
-        "includePattern": ".+\\.js(doc|x)?$"
+        "include": ["lib"],
+        "exclude": ["lib/kafkajs/_common.js"],
+        "includePattern": ".+\\.js?$",
+        "excludePattern": ""
     },
     "sourceType": "module",
     "tags": {
         "allowUnknownTags": true,
         "dictionaries": ["jsdoc","closure"]
     },
     "templates": {
-        "cleverLinks": false,
-        "monospaceLinks": false
+        "default": {
+            "useLongnameInNav": true,
+            "outputSourceFiles": false
+        }
     }
 }

lib/admin.js

@@ -13,16 +13,29 @@
  * hardcoded list.
  * New additions won't be automatically added to this list.
  */
-const ConsumerGroupStates = Object.seal({
+
+/**
+ * A list of consumer group states.
+ * @enum {number}
+ * @readonly
+ * @memberof RdKafka
+ */
+const ConsumerGroupStates = {
   UNKNOWN: 0,
   PREPARING_REBALANCE: 1,
   COMPLETING_REBALANCE: 2,
   STABLE: 3,
   DEAD: 4,
   EMPTY: 5,
-});
+};
 
-const AclOperationTypes = Object.seal({
+/**
+ * A list of ACL operation types.
+ * @enum {number}
+ * @readonly
+ * @memberof RdKafka
+ */
+const AclOperationTypes = {
   UNKNOWN: 0,
   ANY: 1,
   ALL: 2,
@@ -36,7 +49,7 @@ const AclOperationTypes = Object.seal({
   DESCRIBE_CONFIGS: 10,
   ALTER_CONFIGS: 11,
   IDEMPOTENT_WRITE: 12,
-});
+};
 
 /**
  * A list of isolation levels.
@@ -80,8 +93,8 @@ OffsetSpec.LATEST = new OffsetSpec(-1);
 module.exports = {
   create: createAdminClient,
   createFrom: createAdminClientFrom,
-  ConsumerGroupStates,
-  AclOperationTypes,
+  ConsumerGroupStates: Object.freeze(ConsumerGroupStates),
+  AclOperationTypes: Object.freeze(AclOperationTypes),
   IsolationLevel: Object.freeze(IsolationLevel),
   OffsetSpec,
 };
@@ -95,14 +108,15 @@ var { shallowCopy } = require('./util');
 util.inherits(AdminClient, Client);
 
 /**
- * Create a new AdminClient for making topics, partitions, and more.
+ * Create a new AdminClient using the provided configuration.
  *
  * This is a factory method because it immediately starts an
  * active handle with the brokers.
  *
- * @param {object} conf - Key value pairs to configure the admin client
- * @param {object} eventHandlers - optional key value pairs of event handlers to attach to the client
- *
+ * @param {object} conf - Key value pairs to configure the admin client.
+ * @param {object?} eventHandlers - Optional key value pairs of event handlers to attach to the client.
+ * @memberof RdKafka
+ * @alias RdKafka.AdminClient.create
  */
 function createAdminClient(conf, eventHandlers) {
   var client = new AdminClient(conf);
@@ -126,11 +140,14 @@ function createAdminClient(conf, eventHandlers) {
  * This is a factory method because it immediately starts an
  * active handle with the brokers.
  *
- * The producer or consumer being used must be connected.
- * The client can only be used while the producer or consumer is connected.
+ * The producer or consumer being used must be connected before creating the admin client,
+ * and the admin client can only be used while the producer or consumer is connected.
+ *
  * Logging and other events from this client will be emitted on the producer or consumer.
- * @param {import('../types/rdkafka').Producer | import('../types/rdkafka').KafkaConsumer} existingClient a producer or consumer to create the admin client from
- * @param {object} eventHandlers optional key value pairs of event handlers to attach to the client
+ * @param {RdKafka.Producer | RdKafka.KafkaConsumer} existingClient - A producer or consumer to create the admin client from.
+ * @param {object?} eventHandlers - Optional key value pairs of event handlers to attach to the client.
+ * @memberof RdKafka
+ * @alias RdKafka.AdminClient.createFrom
  */
 function createAdminClientFrom(existingClient, eventHandlers) {
   var client = new AdminClient(null, existingClient);
@@ -148,25 +165,25 @@ function createAdminClientFrom(existingClient, eventHandlers) {
 }
 
 /**
- * AdminClient class for administering Kafka
+ * AdminClient class for administering Kafka clusters (promise-based, async API).
  *
  * This client is the way you can interface with the Kafka Admin APIs.
  * This class should not be made using the constructor, but instead
- * should be made using the factory method.
- *
- * <code>
- * var client = AdminClient.create({ ... }); // From configuration
- * var client = AdminClient.createFrom(existingClient); // From existing producer or consumer
- * </code>
+ * should be made using the factory methods (see {@link RdKafka.AdminClient.create}
+ * and {@link RdKafka.AdminClient.createFrom}).
  *
  * Once you instantiate this object, it will have a handle to the kafka broker.
  * Unlike the other confluent-kafka-javascript classes, this class does not ensure that
  * it is connected to the upstream broker. Instead, making an action will
  * validate that.
  *
- * @param {object} conf - Key value pairs to configure the admin client
- * topic configuration
- * @param {import('../types/rdkafka').Producer | import('../types/rdkafka').KafkaConsumer | null} existingClient
+ * @example
+ * var client = AdminClient.create({ ... }); // From configuration
+ * var client = AdminClient.createFrom(existingClient); // From existing producer or consumer
+ *
+ * @param {object|null} conf - Key value pairs to configure the admin client
+ * @param {RdKafka.Producer | RdKafka.KafkaConsumer | null} existingClient - An existing producer or consumer to create the admin client from (optional).
+ * @memberof RdKafka
  * @constructor
  */
 function AdminClient(conf, existingClient) {
@@ -197,6 +214,7 @@ function AdminClient(conf, existingClient) {
  * need to be called outside.
  *
  * Unlike the other connect methods, this one is synchronous.
+ * @private
  */
 AdminClient.prototype.connect = function () {
   if (!this._hasUnderlyingClient) {
@@ -213,7 +231,7 @@ AdminClient.prototype.connect = function () {
  * Disconnect the admin client.
  *
  * This is a synchronous method, but all it does is clean up
- * some memory and shut some threads down
+ * some memory and shut some threads down.
  */
 AdminClient.prototype.disconnect = function () {
   if (this._hasUnderlyingClient) {
@@ -231,9 +249,13 @@ AdminClient.prototype.disconnect = function () {
 /**
  * Create a topic with a given config.
  *
- * @param {NewTopic} topic - Topic to create.
- * @param {number} timeout - Number of milliseconds to wait while trying to create the topic.
- * @param {function} cb - The callback to be executed when finished
+ * @param {object} topic - Topic to create.
+ * @param {string} topic.topic - The name of the topic to create.
+ * @param {number} topic.num_partitions - The number of partitions for the topic.
+ * @param {number} topic.replication_factor - The replication factor for the topic.
+ * @param {object?} topic.config - The topic configuration. The keys of this object denote the keys of the configuration.
+ * @param {number?} timeout - Number of milliseconds to wait while trying to create the topic. Set to 5000 by default.
+ * @param {function} cb - The callback to be executed when finished.
  */
 AdminClient.prototype.createTopic = function (topic, timeout, cb) {
   if (!this._isConnected) {
@@ -267,8 +289,8 @@ AdminClient.prototype.createTopic = function (topic, timeout, cb) {
  * Delete a topic.
  *
  * @param {string} topic - The topic to delete, by name.
- * @param {number} timeout - Number of milliseconds to wait while trying to delete the topic.
- * @param {function} cb - The callback to be executed when finished
+ * @param {number?} timeout - Number of milliseconds to wait while trying to delete the topic. Set to 5000 by default.
+ * @param {function} cb - The callback to be executed when finished.
  */
 AdminClient.prototype.deleteTopic = function (topic, timeout, cb) {
   if (!this._isConnected) {
@@ -303,9 +325,9 @@ AdminClient.prototype.deleteTopic = function (topic, timeout, cb) {
  *
  * @param {string} topic - The topic to add partitions to, by name.
  * @param {number} totalPartitions - The total number of partitions the topic should have
- *                                   after the request
- * @param {number} timeout - Number of milliseconds to wait while trying to create the partitions.
- * @param {function} cb - The callback to be executed when finished
+ *                                   after the request.
+ * @param {number?} timeout - Number of milliseconds to wait while trying to create the partitions. Set to 5000 by default.
+ * @param {function} cb - The callback to be executed when finished.
  */
 AdminClient.prototype.createPartitions = function (topic, totalPartitions, timeout, cb) {
   if (!this._isConnected) {
@@ -337,16 +359,17 @@ AdminClient.prototype.createPartitions = function (topic, totalPartitions, timeo
 
 /**
  * List consumer groups.
+ *
  * @param {any} options
  * @param {number?} options.timeout - The request timeout in milliseconds.
- *                                    May be unset (default: 5000)
- * @param {import("../").ConsumerGroupStates[]?} options.matchConsumerGroupStates -
+ *                                    May be unset (default: 5000).
+ * @param {Array<RdKafka.ConsumerGroupStates>?} options.matchConsumerGroupStates -
  *        A list of consumer group states to match. May be unset, fetches all states (default: unset).
  * @param {function} cb - The callback to be executed when finished.
- *
- * Valid ways to call this function:
- * listGroups(cb)
- * listGroups(options, cb)
+ * @example
+ * // Valid ways to call this function:
+ * listGroups(cb);
+ * listGroups(options, cb);
  */
 AdminClient.prototype.listGroups = function (options, cb) {
   if (!this._isConnected) {
@@ -378,14 +401,15 @@ AdminClient.prototype.listGroups = function (options, cb) {
 
 /**
  * Describe consumer groups.
- * @param {string[]} groups - The names of the groups to describe.
- * @param {any?} options
+ * @param {Array<string>} groups - The names of the groups to describe.
+ * @param {object} options
  * @param {number?} options.timeout - The request timeout in milliseconds.
- *                                    May be unset (default: 5000)
+ *                                    May be unset (default: 5000).
  * @param {boolean?} options.includeAuthorizedOperations - If true, include operations allowed on the group by the calling client (default: false).
  * @param {function} cb - The callback to be executed when finished.
  *
- * Valid ways to call this function:
+ * @example
+ * // Valid ways to call this function:
  * describeGroups(groups, cb)
  * describeGroups(groups, options, cb)
  */
@@ -420,12 +444,13 @@ AdminClient.prototype.describeGroups = function (groups, options, cb) {
 /**
  * Delete consumer groups.
  * @param {string[]} groups - The names of the groups to delete.
- * @param {any?} options
+ * @param {object} options
  * @param {number?} options.timeout - The request timeout in milliseconds.
- *                                    May be unset (default: 5000)
+ *                                    May be unset (default: 5000).
  * @param {function} cb - The callback to be executed when finished.
  *
- * Valid ways to call this function:
+ * @example
+ * // Valid ways to call this function:
  * deleteGroups(groups, cb)
  * deleteGroups(groups, options, cb)
  */
@@ -460,12 +485,13 @@ AdminClient.prototype.deleteGroups = function (groups, options, cb) {
 /**
  * List topics.
  *
- * @param {any?} options
+ * @param {object} options
  * @param {number?} options.timeout - The request timeout in milliseconds.
- *                                    May be unset (default: 5000)
+ *                                    May be unset (default: 5000).
  * @param {function} cb - The callback to be executed when finished.
  *
- * Valid ways to call this function:
+ * @example
+ * // Valid ways to call this function:
  * listTopics(cb)
  * listTopics(options, cb)
  */
@@ -516,13 +542,14 @@ AdminClient.prototype.listTopics = function (options, cb) {
 /**
  * List offsets for topic partition(s) for consumer group(s).
  *
- * @param {import("../../types/rdkafka").ListGroupOffsets} listGroupOffsets - The list of groupId, partitions to fetch offsets for.
- *                                                                            If partitions is null, list offsets for all partitions
- *                                                                            in the group.
+ * @param {{groupId: string, partitions: Array<number>|null}} listGroupOffsets
+ *  The list of groupId, partitions to fetch offsets for. If partitions is null, list offsets for all partitions
+ *  in the group.
+ * @param {object} options
  * @param {number?} options.timeout - The request timeout in milliseconds.
- *                                    May be unset (default: 5000)
+ *                                    May be unset (default: 5000).
  * @param {boolean?} options.requireStableOffsets - Whether broker should return stable offsets
- *                                                  (transaction-committed). (default: false)
+ *                                                  (transaction-committed). (default: false).
  *
  * @param {function} cb - The callback to be executed when finished.
  */
@@ -564,15 +591,15 @@ AdminClient.prototype.listConsumerGroupOffsets = function (listGroupOffsets, opt
 
 /**
  * Deletes records (messages) in topic partitions older than the offsets provided.
- * Provide Topic.OFFSET_END or -1 as offset to delete all records.
+ * Provide Topic.OFFSET_END or -1 as the offset to delete all records.
  *
- * @param {import("../../types/rdkafka").TopicPartitionOffset[]} delRecords - The list of topic partitions and
- *                                                                            offsets to delete records up to.
+ * @param {Array<{topic: string, partition: number, offset: number}>} delRecords
+ * The list of topic partitions and offsets to delete records up to.
+ * @param {object} options
  * @param {number?} options.operationTimeout - The operation timeout in milliseconds.
- *                                             May be unset (default: 60000)
+ *                                             May be unset (default: 60000).
  * @param {number?} options.timeout - The request timeout in milliseconds.
- *                                    May be unset (default: 5000)
- *
+ *                                    May be unset (default: 5000).
  * @param {function} cb - The callback to be executed when finished.
  */
 AdminClient.prototype.deleteRecords = function (delRecords, options, cb) {
@@ -611,13 +638,14 @@ AdminClient.prototype.deleteRecords = function (delRecords, options, cb) {
 };
 
 /**
- * Describe Topics.
+ * Describe topics.
  *
- * @param {string[]} topics - The names of the topics to describe.
+ * @param {Array<string>} topics - The names of the topics to describe.
+ * @param {object} options
  * @param {number?} options.timeout - The request timeout in milliseconds.
- *                                    May be unset (default: 5000)
+ *                                    May be unset (default: 5000).
  * @param {boolean?} options.includeAuthorizedOperations - If true, include operations allowed on the topic by the calling client.
- *                                                         (default: false)
+ *                                                         (default: false).
  * @param {function} cb - The callback to be executed when finished.
  */
 AdminClient.prototype.describeTopics = function (topics, options, cb) {