Open a Modal Window using overlayLibrary

1. Create a lightning component where your modal content will be displayed.

modalContent.cmp

<aura:component access="global">
    <!-- This is the reference of Modal window -->
    <lightning:overlayLibrary aura:id="overlayLibRefId"/>
    <div class="slds-grid slds-wrap">
        <div class="slds-size_8-of-12">
            <div class="slds-text-align_left">
                ::::: Modal window using overlayLibrary :::::
            </div>            
            <div class="slds-align_absolute-center">
                <lightning:button label="Close" 
                                  onclick="{!c.doCloseOperation}" 
                                  variant="brand" 
                                  class="slds-m-top--medium"/>
            </div>
        </div>
    </div>    
</aura:component>

modalContentController.js

({
 doCloseOperation : function(component, event, helper) {
        //Closing the Modal Window
        component.find("overlayLibRefId").notifyClose();
  
 },
})

2. Create second lightning component from where your actual component (as created above) will be called.

modalWindowOverlay.cmp

<aura:component implements="lightning:actionOverride" 
                access="global">
    <!-- overlayLibId is used to show modal window from controller -->
    <lightning:overlayLibrary aura:id="overlayLibId"/>
    <aura:handler name="init" action="{!c.doInit}" value="{!this}"/>
</aura:component>

modalWindowOverlayController.js

({
    doInit : function (component, event, helper) {
        var modalBody;
        $A.createComponent("c:modalContent", {},
                           function(content, status) {
                               if (status === "SUCCESS") {
                                   modalBody = content;
                                   component.find('overlayLibId').showCustomModal({
                                       header: "Modal Header",
                                       body: modalBody,
                                       showCloseButton: true,
                                       //cssClass: "mymodal CSS"
                                       closeCallback: function() {
                                           console.log('Close ......');
                                       }
                                   })
                               }
                           });
    }
})

3. Now, set the modalWindowOverlay.cmp  under any button and then click the button. It will open the modal window.