diff --git a/Makefile b/Makefile index 08d4583b2504..167cc7ec57e6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ VERSION = 4 PATCHLEVEL = 9 -SUBLEVEL = 117 +SUBLEVEL = 118 EXTRAVERSION = NAME = Roaring Lionus diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 7cb107707fed..995cd2428719 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -7085,6 +7085,8 @@ static int handle_vmon(struct kvm_vcpu *vcpu) HRTIMER_MODE_REL_PINNED); vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; + vmx->nested.vpid02 = allocate_vpid(); + vmx->nested.vmxon = true; skip_emulated_instruction(vcpu); @@ -9263,10 +9265,8 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) goto free_vmcs; } - if (nested) { + if (nested) nested_vmx_setup_ctls_msrs(vmx); - vmx->nested.vpid02 = allocate_vpid(); - } vmx->nested.posted_intr_nv = -1; vmx->nested.current_vmptr = -1ull; @@ -9284,7 +9284,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) return &vmx->vcpu; free_vmcs: - free_vpid(vmx->nested.vpid02); free_loaded_vmcs(vmx->loaded_vmcs); free_msrs: kfree(vmx->guest_msrs); diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index 9126627cbf4d..75f2bef79718 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -266,6 +266,8 @@ static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key, return; } + count -= initial; + if (initial) asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ : "+S"(input), "+D"(output) @@ -273,7 +275,7 @@ static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key, asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ : "+S"(input), "+D"(output) - : "d"(control_word), "b"(key), "c"(count - initial)); + : "d"(control_word), "b"(key), "c"(count)); } static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key, @@ -284,6 +286,8 @@ static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key, if (count < cbc_fetch_blocks) return cbc_crypt(input, output, key, iv, control_word, count); + count -= initial; + if (initial) asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ : "+S" (input), "+D" (output), "+a" (iv) @@ -291,7 +295,7 @@ static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key, asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ : "+S" (input), "+D" (output), "+a" (iv) - : "d" (control_word), "b" (key), "c" (count-initial)); + : "d" (control_word), "b" (key), "c" (count)); return iv; } diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c index 75056553b06c..f8c9f6f4f822 100644 --- a/drivers/gpu/drm/vc4/vc4_plane.c +++ b/drivers/gpu/drm/vc4/vc4_plane.c @@ -350,6 +350,9 @@ static int vc4_plane_setup_clipping_and_scaling(struct drm_plane_state *state) vc4_state->x_scaling[0] = VC4_SCALING_TPZ; if (vc4_state->y_scaling[0] == VC4_SCALING_NONE) vc4_state->y_scaling[0] = VC4_SCALING_TPZ; + } else { + vc4_state->x_scaling[1] = VC4_SCALING_NONE; + vc4_state->y_scaling[1] = VC4_SCALING_NONE; } vc4_state->is_unity = (vc4_state->x_scaling[0] == VC4_SCALING_NONE && diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index f5fcc0850dac..8a5e0ae4e4c0 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -1682,6 +1682,8 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev) goto err_upper_unlink; } + bond->nest_level = dev_get_nest_level(bond_dev) + 1; + /* If the mode uses primary, then the following is handled by * bond_change_active_slave(). */ @@ -1729,7 +1731,6 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev) if (bond_mode_uses_xmit_hash(bond)) bond_update_slave_arr(bond, NULL); - bond->nest_level = dev_get_nest_level(bond_dev); netdev_info(bond_dev, "Enslaving %s as %s interface with %s link\n", slave_dev->name, @@ -3359,6 +3360,13 @@ static void bond_fold_stats(struct rtnl_link_stats64 *_res, } } +static int bond_get_nest_level(struct net_device *bond_dev) +{ + struct bonding *bond = netdev_priv(bond_dev); + + return bond->nest_level; +} + static struct rtnl_link_stats64 *bond_get_stats(struct net_device *bond_dev, struct rtnl_link_stats64 *stats) { @@ -3367,7 +3375,7 @@ static struct rtnl_link_stats64 *bond_get_stats(struct net_device *bond_dev, struct list_head *iter; struct slave *slave; - spin_lock(&bond->stats_lock); + spin_lock_nested(&bond->stats_lock, bond_get_nest_level(bond_dev)); memcpy(stats, &bond->bond_stats, sizeof(*stats)); rcu_read_lock(); @@ -4163,6 +4171,7 @@ static const struct net_device_ops bond_netdev_ops = { .ndo_neigh_setup = bond_neigh_setup, .ndo_vlan_rx_add_vid = bond_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = bond_vlan_rx_kill_vid, + .ndo_get_lock_subclass = bond_get_nest_level, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_netpoll_setup = bond_netpoll_setup, .ndo_netpoll_cleanup = bond_netpoll_cleanup, @@ -4655,6 +4664,7 @@ static int bond_init(struct net_device *bond_dev) if (!bond->wq) return -ENOMEM; + bond->nest_level = SINGLE_DEPTH_NESTING; netdev_lockdep_set_classes(bond_dev); list_add_tail(&bond->bond_list, &bn->dev_list); diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index b00358297424..d0846ae9e0e4 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -1071,6 +1071,7 @@ static void ems_usb_disconnect(struct usb_interface *intf) usb_free_urb(dev->intr_urb); kfree(dev->intr_in_buffer); + kfree(dev->tx_msg_buffer); } } diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c index e13c9cd45dc0..bcd993140f84 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_com.c @@ -331,6 +331,7 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev, memset(&io_sq->desc_addr, 0x0, sizeof(struct ena_com_io_desc_addr)); + io_sq->dma_addr_bits = ena_dev->dma_addr_bits; io_sq->desc_entry_size = (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? sizeof(struct ena_eth_io_tx_desc) : diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c index 84c5d296d13e..684835833fe3 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c @@ -877,14 +877,14 @@ static void xgbe_phy_adjust_link(struct xgbe_prv_data *pdata) if (pdata->tx_pause != pdata->phy.tx_pause) { new_state = 1; - pdata->hw_if.config_tx_flow_control(pdata); pdata->tx_pause = pdata->phy.tx_pause; + pdata->hw_if.config_tx_flow_control(pdata); } if (pdata->rx_pause != pdata->phy.rx_pause) { new_state = 1; - pdata->hw_if.config_rx_flow_control(pdata); pdata->rx_pause = pdata->phy.rx_pause; + pdata->hw_if.config_rx_flow_control(pdata); } /* Speed support */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 9bba59ce7477..61f09382c053 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -58,7 +58,7 @@ #ifdef CONFIG_DWMAC_MESON #include #endif -#define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x) +#define STMMAC_ALIGN(x) __ALIGN_KERNEL(x, SMP_CACHE_BYTES) #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) /* Module parameters */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c index 56c8a2342c14..eafc28142cd2 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c @@ -183,7 +183,7 @@ static int stmmac_pci_probe(struct pci_dev *pdev, return -ENOMEM; /* Enable pci device */ - ret = pcim_enable_device(pdev); + ret = pci_enable_device(pdev); if (ret) { dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n", __func__); @@ -232,9 +232,45 @@ static int stmmac_pci_probe(struct pci_dev *pdev, static void stmmac_pci_remove(struct pci_dev *pdev) { stmmac_dvr_remove(&pdev->dev); + pci_disable_device(pdev); } -static SIMPLE_DEV_PM_OPS(stmmac_pm_ops, stmmac_suspend, stmmac_resume); +static int stmmac_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = stmmac_suspend(dev); + if (ret) + return ret; + + ret = pci_save_state(pdev); + if (ret) + return ret; + + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + return 0; +} + +static int stmmac_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pci_restore_state(pdev); + pci_set_power_state(pdev, PCI_D0); + + ret = pci_enable_device(pdev); + if (ret) + return ret; + + pci_set_master(pdev); + + return stmmac_resume(dev); +} + +static SIMPLE_DEV_PM_OPS(stmmac_pm_ops, stmmac_pci_suspend, stmmac_pci_resume); #define STMMAC_VENDOR_ID 0x700 #define STMMAC_QUARK_ID 0x0937 diff --git a/drivers/net/phy/mdio-mux-bcm-iproc.c b/drivers/net/phy/mdio-mux-bcm-iproc.c index 0a5f62e0efcc..487bf5b8f545 100644 --- a/drivers/net/phy/mdio-mux-bcm-iproc.c +++ b/drivers/net/phy/mdio-mux-bcm-iproc.c @@ -218,7 +218,7 @@ out: static int mdio_mux_iproc_remove(struct platform_device *pdev) { - struct iproc_mdiomux_desc *md = dev_get_platdata(&pdev->dev); + struct iproc_mdiomux_desc *md = platform_get_drvdata(pdev); mdio_mux_uninit(md->mux_handle); mdiobus_unregister(md->mii_bus); diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index 5e0626c80b81..c5e04d1ad73a 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -1170,6 +1170,8 @@ static int lan78xx_link_reset(struct lan78xx_net *dev) mod_timer(&dev->stat_monitor, jiffies + STAT_UPDATE_TIMER); } + + tasklet_schedule(&dev->bh); } return ret; diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index a5908e4c06cb..681256f97cb3 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -86,6 +86,7 @@ struct netfront_cb { /* IRQ name is queue name with "-tx" or "-rx" appended */ #define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3) +static DECLARE_WAIT_QUEUE_HEAD(module_load_q); static DECLARE_WAIT_QUEUE_HEAD(module_unload_q); struct netfront_stats { @@ -1349,6 +1350,11 @@ static struct net_device *xennet_create_dev(struct xenbus_device *dev) netif_carrier_off(netdev); xenbus_switch_state(dev, XenbusStateInitialising); + wait_event(module_load_q, + xenbus_read_driver_state(dev->otherend) != + XenbusStateClosed && + xenbus_read_driver_state(dev->otherend) != + XenbusStateUnknown); return netdev; exit: diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c index b40a074822cf..15aeeb2159cc 100644 --- a/drivers/pinctrl/intel/pinctrl-intel.c +++ b/drivers/pinctrl/intel/pinctrl-intel.c @@ -604,12 +604,17 @@ static int intel_gpio_get(struct gpio_chip *chip, unsigned offset) { struct intel_pinctrl *pctrl = gpiochip_get_data(chip); void __iomem *reg; + u32 padcfg0; reg = intel_get_padcfg(pctrl, offset, PADCFG0); if (!reg) return -EINVAL; - return !!(readl(reg) & PADCFG0_GPIORXSTATE); + padcfg0 = readl(reg); + if (!(padcfg0 & PADCFG0_GPIOTXDIS)) + return !!(padcfg0 & PADCFG0_GPIOTXSTATE); + + return !!(padcfg0 & PADCFG0_GPIORXSTATE); } static void intel_gpio_set(struct gpio_chip *chip, unsigned offset, int value) diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 2065a0f9dca6..8d9b416399f9 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -2185,6 +2185,7 @@ sg_add_sfp(Sg_device * sdp) write_lock_irqsave(&sdp->sfd_lock, iflags); if (atomic_read(&sdp->detaching)) { write_unlock_irqrestore(&sdp->sfd_lock, iflags); + kfree(sfp); return ERR_PTR(-ENODEV); } list_add_tail(&sfp->sfd_siblings, &sdp->sfds); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index a7c08cc4c1b7..30076956a096 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -493,7 +493,9 @@ static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info, tell_host(vb, vb->inflate_vq); /* balloon's page migration 2nd step -- deflate "page" */ + spin_lock_irqsave(&vb_dev_info->pages_lock, flags); balloon_page_delete(page); + spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; set_page_pfns(vb, vb->pfns, page); tell_host(vb, vb->deflate_vq); diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig index 6dd158a216f4..ffb093e72b6c 100644 --- a/fs/squashfs/Kconfig +++ b/fs/squashfs/Kconfig @@ -25,6 +25,34 @@ config SQUASHFS If unsure, say N. +choice + prompt "File decompression options" + depends on SQUASHFS + help + Squashfs now supports two options for decompressing file + data. Traditionally Squashfs has decompressed into an + intermediate buffer and then memcopied it into the page cache. + Squashfs now supports the ability to decompress directly into + the page cache. + + If unsure, select "Decompress file data into an intermediate buffer" + +config SQUASHFS_FILE_CACHE + bool "Decompress file data into an intermediate buffer" + help + Decompress file data into an intermediate buffer and then + memcopy it into the page cache. + +config SQUASHFS_FILE_DIRECT + bool "Decompress files directly into the page cache" + help + Directly decompress file data into the page cache. + Doing so can significantly improve performance because + it eliminates a memcpy and it also removes the lock contention + on the single buffer. + +endchoice + choice prompt "Decompressor parallelisation options" depends on SQUASHFS diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile index fe51f1507ed1..246a6f329d89 100644 --- a/fs/squashfs/Makefile +++ b/fs/squashfs/Makefile @@ -5,7 +5,8 @@ obj-$(CONFIG_SQUASHFS) += squashfs.o squashfs-y += block.o cache.o dir.o export.o file.o fragment.o id.o inode.o squashfs-y += namei.o super.o symlink.o decompressor.o -squashfs-y += file_direct.o page_actor.o +squashfs-$(CONFIG_SQUASHFS_FILE_CACHE) += file_cache.o +squashfs-$(CONFIG_SQUASHFS_FILE_DIRECT) += file_direct.o page_actor.o squashfs-$(CONFIG_SQUASHFS_DECOMP_SINGLE) += decompressor_single.o squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI) += decompressor_multi.o squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU) += decompressor_multi_percpu.o diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c index 707747617148..cec0fa208078 100644 --- a/fs/squashfs/block.c +++ b/fs/squashfs/block.c @@ -28,12 +28,9 @@ #include #include -#include #include #include -#include #include -#include #include "squashfs_fs.h" #include "squashfs_fs_sb.h" @@ -41,382 +38,45 @@ #include "decompressor.h" #include "page_actor.h" -static struct workqueue_struct *squashfs_read_wq; - -struct squashfs_read_request { - struct super_block *sb; - u64 index; - int length; - int compressed; - int offset; - u64 read_end; - struct squashfs_page_actor *output; - enum { - SQUASHFS_COPY, - SQUASHFS_DECOMPRESS, - SQUASHFS_METADATA, - } data_processing; - bool synchronous; - - /* - * If the read is synchronous, it is possible to retrieve information - * about the request by setting these pointers. - */ - int *res; - int *bytes_read; - int *bytes_uncompressed; - - int nr_buffers; - struct buffer_head **bh; - struct work_struct offload; -}; - -struct squashfs_bio_request { - struct buffer_head **bh; - int nr_buffers; -}; - -static int squashfs_bio_submit(struct squashfs_read_request *req); - -int squashfs_init_read_wq(void) +/* + * Read the metadata block length, this is stored in the first two + * bytes of the metadata block. + */ +static struct buffer_head *get_block_length(struct super_block *sb, + u64 *cur_index, int *offset, int *length) { - squashfs_read_wq = create_workqueue("SquashFS read wq"); - return !!squashfs_read_wq; -} - -void squashfs_destroy_read_wq(void) -{ - flush_workqueue(squashfs_read_wq); - destroy_workqueue(squashfs_read_wq); -} - -static void free_read_request(struct squashfs_read_request *req, int error) -{ - if (!req->synchronous) - squashfs_page_actor_free(req->output, error); - if (req->res) - *(req->res) = error; - kfree(req->bh); - kfree(req); -} - -static void squashfs_process_blocks(struct squashfs_read_request *req) -{ - int error = 0; - int bytes, i, length; - struct squashfs_sb_info *msblk = req->sb->s_fs_info; - struct squashfs_page_actor *actor = req->output; - struct buffer_head **bh = req->bh; - int nr_buffers = req->nr_buffers; - - for (i = 0; i < nr_buffers; ++i) { - if (!bh[i]) - continue; - wait_on_buffer(bh[i]); - if (!buffer_uptodate(bh[i])) - error = -EIO; - } - if (error) - goto cleanup; - - if (req->data_processing == SQUASHFS_METADATA) { - /* Extract the length of the metadata block */ - if (req->offset != msblk->devblksize - 1) { - length = le16_to_cpup((__le16 *) - (bh[0]->b_data + req->offset)); - } else { - length = (unsigned char)bh[0]->b_data[req->offset]; - length |= (unsigned char)bh[1]->b_data[0] << 8; - } - req->compressed = SQUASHFS_COMPRESSED(length); - req->data_processing = req->compressed ? SQUASHFS_DECOMPRESS - : SQUASHFS_COPY; - length = SQUASHFS_COMPRESSED_SIZE(length); - if (req->index + length + 2 > req->read_end) { - for (i = 0; i < nr_buffers; ++i) - put_bh(bh[i]); - kfree(bh); - req->length = length; - req->index += 2; - squashfs_bio_submit(req); - return; - } - req->length = length; - req->offset = (req->offset + 2) % PAGE_SIZE; - if (req->offset < 2) { - put_bh(bh[0]); - ++bh; - --nr_buffers; - } - } - if (req->bytes_read) - *(req->bytes_read) = req->length; - - if (req->data_processing == SQUASHFS_COPY) { - squashfs_bh_to_actor(bh, nr_buffers, req->output, req->offset, - req->length, msblk->devblksize); - } else if (req->data_processing == SQUASHFS_DECOMPRESS) { - req->length = squashfs_decompress(msblk, bh, nr_buffers, - req->offset, req->length, actor); - if (req->length < 0) { - error = -EIO; - goto cleanup; - } - } - - /* Last page may have trailing bytes not filled */ - bytes = req->length % PAGE_SIZE; - if (bytes && actor->page[actor->pages - 1]) - zero_user_segment(actor->page[actor->pages - 1], bytes, - PAGE_SIZE); - -cleanup: - if (req->bytes_uncompressed) - *(req->bytes_uncompressed) = req->length; - if (error) { - for (i = 0; i < nr_buffers; ++i) - if (bh[i]) - put_bh(bh[i]); - } - free_read_request(req, error); -} - -static void read_wq_handler(struct work_struct *work) -{ - squashfs_process_blocks(container_of(work, - struct squashfs_read_request, offload)); -} - -static void squashfs_bio_end_io(struct bio *bio) -{ - int i; - int error = bio->bi_error; - struct squashfs_bio_request *bio_req = bio->bi_private; - - bio_put(bio); - - for (i = 0; i < bio_req->nr_buffers; ++i) { - if (!bio_req->bh[i]) - continue; - if (!error) - set_buffer_uptodate(bio_req->bh[i]); - else - clear_buffer_uptodate(bio_req->bh[i]); - unlock_buffer(bio_req->bh[i]); - } - kfree(bio_req); -} - -static int bh_is_optional(struct squashfs_read_request *req, int idx) -{ - int start_idx, end_idx; - struct squashfs_sb_info *msblk = req->sb->s_fs_info; - - start_idx = (idx * msblk->devblksize - req->offset) >> PAGE_SHIFT; - end_idx = ((idx + 1) * msblk->devblksize - req->offset + 1) >> PAGE_SHIFT; - if (start_idx >= req->output->pages) - return 1; - if (start_idx < 0) - start_idx = end_idx; - if (end_idx >= req->output->pages) - end_idx = start_idx; - return !req->output->page[start_idx] && !req->output->page[end_idx]; -} - -static int actor_getblks(struct squashfs_read_request *req, u64 block) -{ - int i; - - req->bh = kmalloc_array(req->nr_buffers, sizeof(*(req->bh)), GFP_NOIO); - if (!req->bh) - return -ENOMEM; - - for (i = 0; i < req->nr_buffers; ++i) { - /* - * When dealing with an uncompressed block, the actor may - * contains NULL pages. There's no need to read the buffers - * associated with these pages. - */ - if (!req->compressed && bh_is_optional(req, i)) { - req->bh[i] = NULL; - continue; - } - req->bh[i] = sb_getblk(req->sb, block + i); - if (!req->bh[i]) { - while (--i) { - if (req->bh[i]) - put_bh(req->bh[i]); - } - return -1; - } - } - return 0; -} - -static int squashfs_bio_submit(struct squashfs_read_request *req) -{ - struct bio *bio = NULL; + struct squashfs_sb_info *msblk = sb->s_fs_info; struct buffer_head *bh; - struct squashfs_bio_request *bio_req = NULL; - int b = 0, prev_block = 0; - struct squashfs_sb_info *msblk = req->sb->s_fs_info; - u64 read_start = round_down(req->index, msblk->devblksize); - u64 read_end = round_up(req->index + req->length, msblk->devblksize); - sector_t block = read_start >> msblk->devblksize_log2; - sector_t block_end = read_end >> msblk->devblksize_log2; - int offset = read_start - round_down(req->index, PAGE_SIZE); - int nr_buffers = block_end - block; - int blksz = msblk->devblksize; - int bio_max_pages = nr_buffers > BIO_MAX_PAGES ? BIO_MAX_PAGES - : nr_buffers; + bh = sb_bread(sb, *cur_index); + if (bh == NULL) + return NULL; - /* Setup the request */ - req->read_end = read_end; - req->offset = req->index - read_start; - req->nr_buffers = nr_buffers; - if (actor_getblks(req, block) < 0) - goto getblk_failed; + if (msblk->devblksize - *offset == 1) { + *length = (unsigned char) bh->b_data[*offset]; + put_bh(bh); + bh = sb_bread(sb, ++(*cur_index)); + if (bh == NULL) + return NULL; + *length |= (unsigned char) bh->b_data[0] << 8; + *offset = 1; + } else { + *length = (unsigned char) bh->b_data[*offset] | + (unsigned char) bh->b_data[*offset + 1] << 8; + *offset += 2; - /* Create and submit the BIOs */ - for (b = 0; b < nr_buffers; ++b, offset += blksz) { - bh = req->bh[b]; - if (!bh || !trylock_buffer(bh)) - continue; - if (buffer_uptodate(bh)) { - unlock_buffer(bh); - continue; + if (*offset == msblk->devblksize) { + put_bh(bh); + bh = sb_bread(sb, ++(*cur_index)); + if (bh == NULL) + return NULL; + *offset = 0; } - offset %= PAGE_SIZE; - - /* Append the buffer to the current BIO if it is contiguous */ - if (bio && bio_req && prev_block + 1 == b) { - if (bio_add_page(bio, bh->b_page, blksz, offset)) { - bio_req->nr_buffers += 1; - prev_block = b; - continue; - } - } - - /* Otherwise, submit the current BIO and create a new one */ - if (bio) - submit_bio(bio); - bio_req = kcalloc(1, sizeof(struct squashfs_bio_request), - GFP_NOIO); - if (!bio_req) - goto req_alloc_failed; - bio_req->bh = &req->bh[b]; - bio = bio_alloc(GFP_NOIO, bio_max_pages); - if (!bio) - goto bio_alloc_failed; - bio->bi_bdev = req->sb->s_bdev; - bio->bi_iter.bi_sector = (block + b) - << (msblk->devblksize_log2 - 9); - bio_set_op_attrs(bio, REQ_OP_READ, 0); - bio->bi_private = bio_req; - bio->bi_end_io = squashfs_bio_end_io; - - bio_add_page(bio, bh->b_page, blksz, offset); - bio_req->nr_buffers += 1; - prev_block = b; } - if (bio) - submit_bio(bio); - if (req->synchronous) - squashfs_process_blocks(req); - else { - INIT_WORK(&req->offload, read_wq_handler); - schedule_work(&req->offload); - } - return 0; - -bio_alloc_failed: - kfree(bio_req); -req_alloc_failed: - unlock_buffer(bh); - while (--nr_buffers >= b) - if (req->bh[nr_buffers]) - put_bh(req->bh[nr_buffers]); - while (--b >= 0) - if (req->bh[b]) - wait_on_buffer(req->bh[b]); -getblk_failed: - free_read_request(req, -ENOMEM); - return -ENOMEM; + return bh; } -static int read_metadata_block(struct squashfs_read_request *req, - u64 *next_index) -{ - int ret, error, bytes_read = 0, bytes_uncompressed = 0; - struct squashfs_sb_info *msblk = req->sb->s_fs_info; - - if (req->index + 2 > msblk->bytes_used) { - free_read_request(req, -EINVAL); - return -EINVAL; - } - req->length = 2; - - /* Do not read beyond the end of the device */ - if (req->index + req->length > msblk->bytes_used) - req->length = msblk->bytes_used - req->index; - req->data_processing = SQUASHFS_METADATA; - - /* - * Reading metadata is always synchronous because we don't know the - * length in advance and the function is expected to update - * 'next_index' and return the length. - */ - req->synchronous = true; - req->res = &error; - req->bytes_read = &bytes_read; - req->bytes_uncompressed = &bytes_uncompressed; - - TRACE("Metadata block @ 0x%llx, %scompressed size %d, src size %d\n", - req->index, req->compressed ? "" : "un", bytes_read, - req->output->length); - - ret = squashfs_bio_submit(req); - if (ret) - return ret; - if (error) - return error; - if (next_index) - *next_index += 2 + bytes_read; - return bytes_uncompressed; -} - -static int read_data_block(struct squashfs_read_request *req, int length, - u64 *next_index, bool synchronous) -{ - int ret, error = 0, bytes_uncompressed = 0, bytes_read = 0; - - req->compressed = SQUASHFS_COMPRESSED_BLOCK(length); - req->length = length = SQUASHFS_COMPRESSED_SIZE_BLOCK(length); - req->data_processing = req->compressed ? SQUASHFS_DECOMPRESS - : SQUASHFS_COPY; - - req->synchronous = synchronous; - if (synchronous) { - req->res = &error; - req->bytes_read = &bytes_read; - req->bytes_uncompressed = &bytes_uncompressed; - } - - TRACE("Data block @ 0x%llx, %scompressed size %d, src size %d\n", - req->index, req->compressed ? "" : "un", req->length, - req->output->length); - - ret = squashfs_bio_submit(req); - if (ret) - return ret; - if (synchronous) - ret = error ? error : bytes_uncompressed; - if (next_index) - *next_index += length; - return ret; -} /* * Read and decompress a metadata block or datablock. Length is non-zero @@ -427,50 +87,130 @@ static int read_data_block(struct squashfs_read_request *req, int length, * generated a larger block - this does occasionally happen with compression * algorithms). */ -static int __squashfs_read_data(struct super_block *sb, u64 index, int length, - u64 *next_index, struct squashfs_page_actor *output, bool sync) -{ - struct squashfs_read_request *req; - - req = kcalloc(1, sizeof(struct squashfs_read_request), GFP_KERNEL); - if (!req) { - if (!sync) - squashfs_page_actor_free(output, -ENOMEM); - return -ENOMEM; - } - - req->sb = sb; - req->index = index; - req->output = output; - - if (next_index) - *next_index = index; - - if (length) - length = read_data_block(req, length, next_index, sync); - else - length = read_metadata_block(req, next_index); - - if (length < 0) { - ERROR("squashfs_read_data failed to read block 0x%llx\n", - (unsigned long long)index); - return -EIO; - } - - return length; -} - int squashfs_read_data(struct super_block *sb, u64 index, int length, - u64 *next_index, struct squashfs_page_actor *output) + u64 *next_index, struct squashfs_page_actor *output) { - return __squashfs_read_data(sb, index, length, next_index, output, - true); -} + struct squashfs_sb_info *msblk = sb->s_fs_info; + struct buffer_head **bh; + int offset = index & ((1 << msblk->devblksize_log2) - 1); + u64 cur_index = index >> msblk->devblksize_log2; + int bytes, compressed, b = 0, k = 0, avail, i; -int squashfs_read_data_async(struct super_block *sb, u64 index, int length, - u64 *next_index, struct squashfs_page_actor *output) -{ + bh = kcalloc(((output->length + msblk->devblksize - 1) + >> msblk->devblksize_log2) + 1, sizeof(*bh), GFP_KERNEL); + if (bh == NULL) + return -ENOMEM; - return __squashfs_read_data(sb, index, length, next_index, output, - false); + if (length) { + /* + * Datablock. + */ + bytes = -offset; + compressed = SQUASHFS_COMPRESSED_BLOCK(length); + length = SQUASHFS_COMPRESSED_SIZE_BLOCK(length); + if (next_index) + *next_index = index + length; + + TRACE("Block @ 0x%llx, %scompressed size %d, src size %d\n", + index, compressed ? "" : "un", length, output->length); + + if (length < 0 || length > output->length || + (index + length) > msblk->bytes_used) + goto read_failure; + + for (b = 0; bytes < length; b++, cur_index++) { + bh[b] = sb_getblk(sb, cur_index); + if (bh[b] == NULL) + goto block_release; + bytes += msblk->devblksize; + } + ll_rw_block(REQ_OP_READ, 0, b, bh); + } else { + /* + * Metadata block. + */ + if ((index + 2) > msblk->bytes_used) + goto read_failure; + + bh[0] = get_block_length(sb, &cur_index, &offset, &length); + if (bh[0] == NULL) + goto read_failure; + b = 1; + + bytes = msblk->devblksize - offset; + compressed = SQUASHFS_COMPRESSED(length); + length = SQUASHFS_COMPRESSED_SIZE(length); + if (next_index) + *next_index = index + length + 2; + + TRACE("Block @ 0x%llx, %scompressed size %d\n", index, + compressed ? "" : "un", length); + + if (length < 0 || length > output->length || + (index + length) > msblk->bytes_used) + goto block_release; + + for (; bytes < length; b++) { + bh[b] = sb_getblk(sb, ++cur_index); + if (bh[b] == NULL) + goto block_release; + bytes += msblk->devblksize; + } + ll_rw_block(REQ_OP_READ, 0, b - 1, bh + 1); + } + + for (i = 0; i < b; i++) { + wait_on_buffer(bh[i]); + if (!buffer_uptodate(bh[i])) + goto block_release; + } + + if (compressed) { + if (!msblk->stream) + goto read_failure; + length = squashfs_decompress(msblk, bh, b, offset, length, + output); + if (length < 0) + goto read_failure; + } else { + /* + * Block is uncompressed. + */ + int in, pg_offset = 0; + void *data = squashfs_first_page(output); + + for (bytes = length; k < b; k++) { + in = min(bytes, msblk->devblksize - offset); + bytes -= in; + while (in) { + if (pg_offset == PAGE_SIZE) { + data = squashfs_next_page(output); + pg_offset = 0; + } + avail = min_t(int, in, PAGE_SIZE - + pg_offset); + memcpy(data + pg_offset, bh[k]->b_data + offset, + avail); + in -= avail; + pg_offset += avail; + offset += avail; + } + offset = 0; + put_bh(bh[k]); + } + squashfs_finish_page(output); + } + + kfree(bh); + return length; + +block_release: + for (; k < b; k++) + put_bh(bh[k]); + +read_failure: + ERROR("squashfs_read_data failed to read block 0x%llx\n", + (unsigned long long) index); + kfree(bh); + return -EIO; } diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c index 9d9d4aa9a7ba..0839efa720b3 100644 --- a/fs/squashfs/cache.c +++ b/fs/squashfs/cache.c @@ -209,14 +209,17 @@ void squashfs_cache_put(struct squashfs_cache_entry *entry) */ void squashfs_cache_delete(struct squashfs_cache *cache) { - int i; + int i, j; if (cache == NULL) return; for (i = 0; i < cache->entries; i++) { - if (cache->entry[i].page) - free_page_array(cache->entry[i].page, cache->pages); + if (cache->entry[i].data) { + for (j = 0; j < cache->pages; j++) + kfree(cache->entry[i].data[j]); + kfree(cache->entry[i].data); + } kfree(cache->entry[i].actor); } @@ -233,7 +236,7 @@ void squashfs_cache_delete(struct squashfs_cache *cache) struct squashfs_cache *squashfs_cache_init(char *name, int entries, int block_size) { - int i; + int i, j; struct squashfs_cache *cache = kzalloc(sizeof(*cache), GFP_KERNEL); if (cache == NULL) { @@ -265,13 +268,22 @@ struct squashfs_cache *squashfs_cache_init(char *name, int entries, init_waitqueue_head(&cache->entry[i].wait_queue); entry->cache = cache; entry->block = SQUASHFS_INVALID_BLK; - entry->page = alloc_page_array(cache->pages, GFP_KERNEL); - if (!entry->page) { + entry->data = kcalloc(cache->pages, sizeof(void *), GFP_KERNEL); + if (entry->data == NULL) { ERROR("Failed to allocate %s cache entry\n", name); goto cleanup; } - entry->actor = squashfs_page_actor_init(entry->page, - cache->pages, 0, NULL); + + for (j = 0; j < cache->pages; j++) { + entry->data[j] = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (entry->data[j] == NULL) { + ERROR("Failed to allocate %s buffer\n", name); + goto cleanup; + } + } + + entry->actor = squashfs_page_actor_init(entry->data, + cache->pages, 0); if (entry->actor == NULL) { ERROR("Failed to allocate %s cache entry\n", name); goto cleanup; @@ -302,20 +314,18 @@ int squashfs_copy_data(void *buffer, struct squashfs_cache_entry *entry, return min(length, entry->length - offset); while (offset < entry->length) { - void *buff = kmap_atomic(entry->page[offset / PAGE_SIZE]) - + (offset % PAGE_SIZE); + void *buff = entry->data[offset / PAGE_SIZE] + + (offset % PAGE_SIZE); int bytes = min_t(int, entry->length - offset, PAGE_SIZE - (offset % PAGE_SIZE)); if (bytes >= remaining) { memcpy(buffer, buff, remaining); - kunmap_atomic(buff); remaining = 0; break; } memcpy(buffer, buff, bytes); - kunmap_atomic(buff); buffer += bytes; remaining -= bytes; offset += bytes; @@ -409,38 +419,43 @@ struct squashfs_cache_entry *squashfs_get_datablock(struct super_block *sb, void *squashfs_read_table(struct super_block *sb, u64 block, int length) { int pages = (length + PAGE_SIZE - 1) >> PAGE_SHIFT; - struct page **page; - void *buff; - int res; + int i, res; + void *table, *buffer, **data; struct squashfs_page_actor *actor; - page = alloc_page_array(pages, GFP_KERNEL); - if (!page) + table = buffer = kmalloc(length, GFP_KERNEL); + if (table == NULL) return ERR_PTR(-ENOMEM); - actor = squashfs_page_actor_init(page, pages, length, NULL); - if (actor == NULL) { + data = kcalloc(pages, sizeof(void *), GFP_KERNEL); + if (data == NULL) { res = -ENOMEM; goto failed; } + actor = squashfs_page_actor_init(data, pages, length); + if (actor == NULL) { + res = -ENOMEM; + goto failed2; + } + + for (i = 0; i < pages; i++, buffer += PAGE_SIZE) + data[i] = buffer; + res = squashfs_read_data(sb, block, length | SQUASHFS_COMPRESSED_BIT_BLOCK, NULL, actor); - if (res < 0) - goto failed2; + kfree(data); + kfree(actor); - buff = kmalloc(length, GFP_KERNEL); - if (!buff) - goto failed2; - squashfs_actor_to_buf(actor, buff, length); - squashfs_page_actor_free(actor, 0); - free_page_array(page, pages); - return buff; + if (res < 0) + goto failed; + + return table; failed2: - squashfs_page_actor_free(actor, 0); + kfree(data); failed: - free_page_array(page, pages); + kfree(table); return ERR_PTR(res); } diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c index 7de35bf297aa..d2bc13636f79 100644 --- a/fs/squashfs/decompressor.c +++ b/fs/squashfs/decompressor.c @@ -24,8 +24,7 @@ #include #include #include -#include -#include +#include #include "squashfs_fs.h" #include "squashfs_fs_sb.h" @@ -95,44 +94,40 @@ const struct squashfs_decompressor *squashfs_lookup_decompressor(int id) static void *get_comp_opts(struct super_block *sb, unsigned short flags) { struct squashfs_sb_info *msblk = sb->s_fs_info; - void *comp_opts, *buffer = NULL; - struct page *page; + void *buffer = NULL, *comp_opts; struct squashfs_page_actor *actor = NULL; int length = 0; - if (!SQUASHFS_COMP_OPTS(flags)) - return squashfs_comp_opts(msblk, buffer, length); - /* * Read decompressor specific options from file system if present */ + if (SQUASHFS_COMP_OPTS(flags)) { + buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (buffer == NULL) { + comp_opts = ERR_PTR(-ENOMEM); + goto out; + } - page = alloc_page(GFP_KERNEL); - if (!page) - return ERR_PTR(-ENOMEM); + actor = squashfs_page_actor_init(&buffer, 1, 0); + if (actor == NULL) { + comp_opts = ERR_PTR(-ENOMEM); + goto out; + } - actor = squashfs_page_actor_init(&page, 1, 0, NULL); - if (actor == NULL) { - comp_opts = ERR_PTR(-ENOMEM); - goto actor_error; + length = squashfs_read_data(sb, + sizeof(struct squashfs_super_block), 0, NULL, actor); + + if (length < 0) { + comp_opts = ERR_PTR(length); + goto out; + } } - length = squashfs_read_data(sb, - sizeof(struct squashfs_super_block), 0, NULL, actor); - - if (length < 0) { - comp_opts = ERR_PTR(length); - goto read_error; - } - - buffer = kmap_atomic(page); comp_opts = squashfs_comp_opts(msblk, buffer, length); - kunmap_atomic(buffer); -read_error: - squashfs_page_actor_free(actor, 0); -actor_error: - __free_page(page); +out: + kfree(actor); + kfree(buffer); return comp_opts; } diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c index cd3c5c8211a5..fcff2e0487fe 100644 --- a/fs/squashfs/file.c +++ b/fs/squashfs/file.c @@ -47,7 +47,6 @@ #include #include #include -#include #include "squashfs_fs.h" #include "squashfs_fs_sb.h" @@ -443,21 +442,6 @@ static int squashfs_readpage_fragment(struct page *page) return res; } -static int squashfs_readpages_fragment(struct page *page, - struct list_head *readahead_pages, struct address_space *mapping) -{ - if (!page) { - page = lru_to_page(readahead_pages); - list_del(&page->lru); - if (add_to_page_cache_lru(page, mapping, page->index, - mapping_gfp_constraint(mapping, GFP_KERNEL))) { - put_page(page); - return 0; - } - } - return squashfs_readpage_fragment(page); -} - static int squashfs_readpage_sparse(struct page *page, int index, int file_end) { struct inode *inode = page->mapping->host; @@ -470,105 +454,54 @@ static int squashfs_readpage_sparse(struct page *page, int index, int file_end) return 0; } -static int squashfs_readpages_sparse(struct page *page, - struct list_head *readahead_pages, int index, int file_end, - struct address_space *mapping) -{ - if (!page) { - page = lru_to_page(readahead_pages); - list_del(&page->lru); - if (add_to_page_cache_lru(page, mapping, page->index, - mapping_gfp_constraint(mapping, GFP_KERNEL))) { - put_page(page); - return 0; - } - } - return squashfs_readpage_sparse(page, index, file_end); -} - -static int __squashfs_readpages(struct file *file, struct page *page, - struct list_head *readahead_pages, unsigned int nr_pages, - struct address_space *mapping) -{ - struct inode *inode = mapping->host; - struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; - int file_end = i_size_read(inode) >> msblk->block_log; - int res; - - do { - struct page *cur_page = page ? page - : lru_to_page(readahead_pages); - int page_index = cur_page->index; - int index = page_index >> (msblk->block_log - PAGE_SHIFT); - - if (page_index >= ((i_size_read(inode) + PAGE_SIZE - 1) >> - PAGE_SHIFT)) - return 1; - - if (index < file_end || squashfs_i(inode)->fragment_block == - SQUASHFS_INVALID_BLK) { - u64 block = 0; - int bsize = read_blocklist(inode, index, &block); - - if (bsize < 0) - return -1; - - if (bsize == 0) { - res = squashfs_readpages_sparse(page, - readahead_pages, index, file_end, - mapping); - } else { - res = squashfs_readpages_block(page, - readahead_pages, &nr_pages, mapping, - page_index, block, bsize); - } - } else { - res = squashfs_readpages_fragment(page, - readahead_pages, mapping); - } - if (res) - return 0; - page = NULL; - } while (readahead_pages && !list_empty(readahead_pages)); - - return 0; -} - static int squashfs_readpage(struct file *file, struct page *page) { - int ret; + struct inode *inode = page->mapping->host; + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + int index = page->index >> (msblk->block_log - PAGE_SHIFT); + int file_end = i_size_read(inode) >> msblk->block_log; + int res; + void *pageaddr; TRACE("Entered squashfs_readpage, page index %lx, start block %llx\n", - page->index, squashfs_i(page->mapping->host)->start); + page->index, squashfs_i(inode)->start); - get_page(page); + if (page->index >= ((i_size_read(inode) + PAGE_SIZE - 1) >> + PAGE_SHIFT)) + goto out; - ret = __squashfs_readpages(file, page, NULL, 1, page->mapping); - if (ret) { - flush_dcache_page(page); - if (ret < 0) - SetPageError(page); + if (index < file_end || squashfs_i(inode)->fragment_block == + SQUASHFS_INVALID_BLK) { + u64 block = 0; + int bsize = read_blocklist(inode, index, &block); + if (bsize < 0) + goto error_out; + + if (bsize == 0) + res = squashfs_readpage_sparse(page, index, file_end); else - SetPageUptodate(page); - zero_user_segment(page, 0, PAGE_SIZE); - unlock_page(page); - put_page(page); - } + res = squashfs_readpage_block(page, block, bsize); + } else + res = squashfs_readpage_fragment(page); - return 0; -} + if (!res) + return 0; + +error_out: + SetPageError(page); +out: + pageaddr = kmap_atomic(page); + memset(pageaddr, 0, PAGE_SIZE); + kunmap_atomic(pageaddr); + flush_dcache_page(page); + if (!PageError(page)) + SetPageUptodate(page); + unlock_page(page); -static int squashfs_readpages(struct file *file, struct address_space *mapping, - struct list_head *pages, unsigned int nr_pages) -{ - TRACE("Entered squashfs_readpages, %u pages, first page index %lx\n", - nr_pages, lru_to_page(pages)->index); - __squashfs_readpages(file, NULL, pages, nr_pages, mapping); return 0; } const struct address_space_operations squashfs_aops = { - .readpage = squashfs_readpage, - .readpages = squashfs_readpages, + .readpage = squashfs_readpage }; diff --git a/fs/squashfs/file_cache.c b/fs/squashfs/file_cache.c new file mode 100644 index 000000000000..f2310d2a2019 --- /dev/null +++ b/fs/squashfs/file_cache.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2013 + * Phillip Lougher + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" + +/* Read separately compressed datablock and memcopy into page cache */ +int squashfs_readpage_block(struct page *page, u64 block, int bsize) +{ + struct inode *i = page->mapping->host; + struct squashfs_cache_entry *buffer = squashfs_get_datablock(i->i_sb, + block, bsize); + int res = buffer->error; + + if (res) + ERROR("Unable to read page, block %llx, size %x\n", block, + bsize); + else + squashfs_copy_cache(page, buffer, buffer->length, 0); + + squashfs_cache_put(buffer); + return res; +} diff --git a/fs/squashfs/file_direct.c b/fs/squashfs/file_direct.c index dc87f77ce11e..cb485d8e0e91 100644 --- a/fs/squashfs/file_direct.c +++ b/fs/squashfs/file_direct.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "squashfs_fs.h" #include "squashfs_fs_sb.h" @@ -21,136 +20,157 @@ #include "squashfs.h" #include "page_actor.h" -static void release_actor_pages(struct page **page, int pages, int error) -{ - int i; +static int squashfs_read_cache(struct page *target_page, u64 block, int bsize, + int pages, struct page **page); - for (i = 0; i < pages; i++) { - if (!page[i]) - continue; - flush_dcache_page(page[i]); - if (!error) - SetPageUptodate(page[i]); - else { - SetPageError(page[i]); - zero_user_segment(page[i], 0, PAGE_SIZE); - } - unlock_page(page[i]); - put_page(page[i]); - } - kfree(page); -} +/* Read separately compressed datablock directly into page cache */ +int squashfs_readpage_block(struct page *target_page, u64 block, int bsize) -/* - * Create a "page actor" which will kmap and kunmap the - * page cache pages appropriately within the decompressor - */ -static struct squashfs_page_actor *actor_from_page_cache( - unsigned int actor_pages, struct page *target_page, - struct list_head *rpages, unsigned int *nr_pages, int start_index, - struct address_space *mapping) { + struct inode *inode = target_page->mapping->host; + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + + int file_end = (i_size_read(inode) - 1) >> PAGE_SHIFT; + int mask = (1 << (msblk->block_log - PAGE_SHIFT)) - 1; + int start_index = target_page->index & ~mask; + int end_index = start_index | mask; + int i, n, pages, missing_pages, bytes, res = -ENOMEM; struct page **page; struct squashfs_page_actor *actor; - int i, n; - gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL); + void *pageaddr; - page = kmalloc_array(actor_pages, sizeof(void *), GFP_KERNEL); - if (!page) - return NULL; + if (end_index > file_end) + end_index = file_end; - for (i = 0, n = start_index; i < actor_pages; i++, n++) { - if (target_page == NULL && rpages && !list_empty(rpages)) { - struct page *cur_page = lru_to_page(rpages); + pages = end_index - start_index + 1; - if (cur_page->index < start_index + actor_pages) { - list_del(&cur_page->lru); - --(*nr_pages); - if (add_to_page_cache_lru(cur_page, mapping, - cur_page->index, gfp)) - put_page(cur_page); - else - target_page = cur_page; - } else - rpages = NULL; - } + page = kmalloc_array(pages, sizeof(void *), GFP_KERNEL); + if (page == NULL) + return res; - if (target_page && target_page->index == n) { - page[i] = target_page; - target_page = NULL; - } else { - page[i] = grab_cache_page_nowait(mapping, n); - if (page[i] == NULL) - continue; + /* + * Create a "page actor" which will kmap and kunmap the + * page cache pages appropriately within the decompressor + */ + actor = squashfs_page_actor_init_special(page, pages, 0); + if (actor == NULL) + goto out; + + /* Try to grab all the pages covered by the Squashfs block */ + for (missing_pages = 0, i = 0, n = start_index; i < pages; i++, n++) { + page[i] = (n == target_page->index) ? target_page : + grab_cache_page_nowait(target_page->mapping, n); + + if (page[i] == NULL) { + missing_pages++; + continue; } if (PageUptodate(page[i])) { unlock_page(page[i]); put_page(page[i]); page[i] = NULL; + missing_pages++; } } - actor = squashfs_page_actor_init(page, actor_pages, 0, - release_actor_pages); - if (!actor) { - release_actor_pages(page, actor_pages, -ENOMEM); - kfree(page); - return NULL; + if (missing_pages) { + /* + * Couldn't get one or more pages, this page has either + * been VM reclaimed, but others are still in the page cache + * and uptodate, or we're racing with another thread in + * squashfs_readpage also trying to grab them. Fall back to + * using an intermediate buffer. + */ + res = squashfs_read_cache(target_page, block, bsize, pages, + page); + if (res < 0) + goto mark_errored; + + goto out; } - return actor; -} -int squashfs_readpages_block(struct page *target_page, - struct list_head *readahead_pages, - unsigned int *nr_pages, - struct address_space *mapping, - int page_index, u64 block, int bsize) + /* Decompress directly into the page cache buffers */ + res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor); + if (res < 0) + goto mark_errored; -{ - struct squashfs_page_actor *actor; - struct inode *inode = mapping->host; - struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; - int start_index, end_index, file_end, actor_pages, res; - int mask = (1 << (msblk->block_log - PAGE_SHIFT)) - 1; + /* Last page may have trailing bytes not filled */ + bytes = res % PAGE_SIZE; + if (bytes) { + pageaddr = kmap_atomic(page[pages - 1]); + memset(pageaddr + bytes, 0, PAGE_SIZE - bytes); + kunmap_atomic(pageaddr); + } - /* - * If readpage() is called on an uncompressed datablock, we can just - * read the pages instead of fetching the whole block. - * This greatly improves the performance when a process keep doing - * random reads because we only fetch the necessary data. - * The readahead algorithm will take care of doing speculative reads - * if necessary. - * We can't read more than 1 block even if readahead provides use more - * pages because we don't know yet if the next block is compressed or - * not. + /* Mark pages as uptodate, unlock and release */ + for (i = 0; i < pages; i++) { + flush_dcache_page(page[i]); + SetPageUptodate(page[i]); + unlock_page(page[i]); + if (page[i] != target_page) + put_page(page[i]); + } + + kfree(actor); + kfree(page); + + return 0; + +mark_errored: + /* Decompression failed, mark pages as errored. Target_page is + * dealt with by the caller */ - if (bsize && !SQUASHFS_COMPRESSED_BLOCK(bsize)) { - u64 block_end = block + msblk->block_size; - - block += (page_index & mask) * PAGE_SIZE; - actor_pages = (block_end - block) / PAGE_SIZE; - if (*nr_pages < actor_pages) - actor_pages = *nr_pages; - start_index = page_index; - bsize = min_t(int, bsize, (PAGE_SIZE * actor_pages) - | SQUASHFS_COMPRESSED_BIT_BLOCK); - } else { - file_end = (i_size_read(inode) - 1) >> PAGE_SHIFT; - start_index = page_index & ~mask; - end_index = start_index | mask; - if (end_index > file_end) - end_index = file_end; - actor_pages = end_index - start_index + 1; + for (i = 0; i < pages; i++) { + if (page[i] == NULL || page[i] == target_page) + continue; + flush_dcache_page(page[i]); + SetPageError(page[i]); + unlock_page(page[i]); + put_page(page[i]); } - actor = actor_from_page_cache(actor_pages, target_page, - readahead_pages, nr_pages, start_index, - mapping); - if (!actor) - return -ENOMEM; - - res = squashfs_read_data_async(inode->i_sb, block, bsize, NULL, - actor); - return res < 0 ? res : 0; +out: + kfree(actor); + kfree(page); + return res; +} + + +static int squashfs_read_cache(struct page *target_page, u64 block, int bsize, + int pages, struct page **page) +{ + struct inode *i = target_page->mapping->host; + struct squashfs_cache_entry *buffer = squashfs_get_datablock(i->i_sb, + block, bsize); + int bytes = buffer->length, res = buffer->error, n, offset = 0; + void *pageaddr; + + if (res) { + ERROR("Unable to read page, block %llx, size %x\n", block, + bsize); + goto out; + } + + for (n = 0; n < pages && bytes > 0; n++, + bytes -= PAGE_SIZE, offset += PAGE_SIZE) { + int avail = min_t(int, bytes, PAGE_SIZE); + + if (page[n] == NULL) + continue; + + pageaddr = kmap_atomic(page[n]); + squashfs_copy_data(pageaddr, buffer, offset, avail); + memset(pageaddr + avail, 0, PAGE_SIZE - avail); + kunmap_atomic(pageaddr); + flush_dcache_page(page[n]); + SetPageUptodate(page[n]); + unlock_page(page[n]); + if (page[n] != target_page) + put_page(page[n]); + } + +out: + squashfs_cache_put(buffer); + return res; } diff --git a/fs/squashfs/fragment.c b/fs/squashfs/fragment.c index 86ad9a4b8c36..0681feab4a84 100644 --- a/fs/squashfs/fragment.c +++ b/fs/squashfs/fragment.c @@ -49,11 +49,16 @@ int squashfs_frag_lookup(struct super_block *sb, unsigned int fragment, u64 *fragment_block) { struct squashfs_sb_info *msblk = sb->s_fs_info; - int block = SQUASHFS_FRAGMENT_INDEX(fragment); - int offset = SQUASHFS_FRAGMENT_INDEX_OFFSET(fragment); - u64 start_block = le64_to_cpu(msblk->fragment_index[block]); + int block, offset, size; struct squashfs_fragment_entry fragment_entry; - int size; + u64 start_block; + + if (fragment >= msblk->fragments) + return -EIO; + block = SQUASHFS_FRAGMENT_INDEX(fragment); + offset = SQUASHFS_FRAGMENT_INDEX_OFFSET(fragment); + + start_block = le64_to_cpu(msblk->fragment_index[block]); size = squashfs_read_metadata(sb, &fragment_entry, &start_block, &offset, sizeof(fragment_entry)); diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c index df4fa3c7ddd0..ff4468bd18b0 100644 --- a/fs/squashfs/lz4_wrapper.c +++ b/fs/squashfs/lz4_wrapper.c @@ -94,17 +94,39 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm, struct buffer_head **bh, int b, int offset, int length, struct squashfs_page_actor *output) { - int res; - size_t dest_len = output->length; struct squashfs_lz4 *stream = strm; + void *buff = stream->input, *data; + int avail, i, bytes = length, res; + size_t dest_len = output->length; + + for (i = 0; i < b; i++) { + avail = min(bytes, msblk->devblksize - offset); + memcpy(buff, bh[i]->b_data + offset, avail); + buff += avail; + bytes -= avail; + offset = 0; + put_bh(bh[i]); + } - squashfs_bh_to_buf(bh, b, stream->input, offset, length, - msblk->devblksize); res = lz4_decompress_unknownoutputsize(stream->input, length, stream->output, &dest_len); if (res) return -EIO; - squashfs_buf_to_actor(stream->output, output, dest_len); + + bytes = dest_len; + data = squashfs_first_page(output); + buff = stream->output; + while (data) { + if (bytes <= PAGE_SIZE) { + memcpy(data, buff, bytes); + break; + } + memcpy(data, buff, PAGE_SIZE); + buff += PAGE_SIZE; + bytes -= PAGE_SIZE; + data = squashfs_next_page(output); + } + squashfs_finish_page(output); return dest_len; } diff --git a/fs/squashfs/lzo_wrapper.c b/fs/squashfs/lzo_wrapper.c index 2c844d53a59e..934c17e96590 100644 --- a/fs/squashfs/lzo_wrapper.c +++ b/fs/squashfs/lzo_wrapper.c @@ -79,19 +79,45 @@ static int lzo_uncompress(struct squashfs_sb_info *msblk, void *strm, struct buffer_head **bh, int b, int offset, int length, struct squashfs_page_actor *output) { - int res; - size_t out_len = output->length; struct squashfs_lzo *stream = strm; + void *buff = stream->input, *data; + int avail, i, bytes = length, res; + size_t out_len = output->length; + + for (i = 0; i < b; i++) { + avail = min(bytes, msblk->devblksize - offset); + memcpy(buff, bh[i]->b_data + offset, avail); + buff += avail; + bytes -= avail; + offset = 0; + put_bh(bh[i]); + } - squashfs_bh_to_buf(bh, b, stream->input, offset, length, - msblk->devblksize); res = lzo1x_decompress_safe(stream->input, (size_t)length, stream->output, &out_len); if (res != LZO_E_OK) - return -EIO; - squashfs_buf_to_actor(stream->output, output, out_len); + goto failed; - return out_len; + res = bytes = (int)out_len; + data = squashfs_first_page(output); + buff = stream->output; + while (data) { + if (bytes <= PAGE_SIZE) { + memcpy(data, buff, bytes); + break; + } else { + memcpy(data, buff, PAGE_SIZE); + buff += PAGE_SIZE; + bytes -= PAGE_SIZE; + data = squashfs_next_page(output); + } + } + squashfs_finish_page(output); + + return res; + +failed: + return -EIO; } const struct squashfs_decompressor squashfs_lzo_comp_ops = { diff --git a/fs/squashfs/page_actor.c b/fs/squashfs/page_actor.c index e348f5647fbd..9b7b1b6a7892 100644 --- a/fs/squashfs/page_actor.c +++ b/fs/squashfs/page_actor.c @@ -9,11 +9,79 @@ #include #include #include -#include #include "page_actor.h" -struct squashfs_page_actor *squashfs_page_actor_init(struct page **page, - int pages, int length, void (*release_pages)(struct page **, int, int)) +/* + * This file contains implementations of page_actor for decompressing into + * an intermediate buffer, and for decompressing directly into the + * page cache. + * + * Calling code should avoid sleeping between calls to squashfs_first_page() + * and squashfs_finish_page(). + */ + +/* Implementation of page_actor for decompressing into intermediate buffer */ +static void *cache_first_page(struct squashfs_page_actor *actor) +{ + actor->next_page = 1; + return actor->buffer[0]; +} + +static void *cache_next_page(struct squashfs_page_actor *actor) +{ + if (actor->next_page == actor->pages) + return NULL; + + return actor->buffer[actor->next_page++]; +} + +static void cache_finish_page(struct squashfs_page_actor *actor) +{ + /* empty */ +} + +struct squashfs_page_actor *squashfs_page_actor_init(void **buffer, + int pages, int length) +{ + struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL); + + if (actor == NULL) + return NULL; + + actor->length = length ? : pages * PAGE_SIZE; + actor->buffer = buffer; + actor->pages = pages; + actor->next_page = 0; + actor->squashfs_first_page = cache_first_page; + actor->squashfs_next_page = cache_next_page; + actor->squashfs_finish_page = cache_finish_page; + return actor; +} + +/* Implementation of page_actor for decompressing directly into page cache. */ +static void *direct_first_page(struct squashfs_page_actor *actor) +{ + actor->next_page = 1; + return actor->pageaddr = kmap_atomic(actor->page[0]); +} + +static void *direct_next_page(struct squashfs_page_actor *actor) +{ + if (actor->pageaddr) + kunmap_atomic(actor->pageaddr); + + return actor->pageaddr = actor->next_page == actor->pages ? NULL : + kmap_atomic(actor->page[actor->next_page++]); +} + +static void direct_finish_page(struct squashfs_page_actor *actor) +{ + if (actor->pageaddr) + kunmap_atomic(actor->pageaddr); +} + +struct squashfs_page_actor *squashfs_page_actor_init_special(struct page **page, + int pages, int length) { struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL); @@ -25,129 +93,8 @@ struct squashfs_page_actor *squashfs_page_actor_init(struct page **page, actor->pages = pages; actor->next_page = 0; actor->pageaddr = NULL; - actor->release_pages = release_pages; + actor->squashfs_first_page = direct_first_page; + actor->squashfs_next_page = direct_next_page; + actor->squashfs_finish_page = direct_finish_page; return actor; } - -void squashfs_page_actor_free(struct squashfs_page_actor *actor, int error) -{ - if (!actor) - return; - - if (actor->release_pages) - actor->release_pages(actor->page, actor->pages, error); - kfree(actor); -} - -void squashfs_actor_to_buf(struct squashfs_page_actor *actor, void *buf, - int length) -{ - void *pageaddr; - int pos = 0, avail, i; - - for (i = 0; i < actor->pages && pos < length; ++i) { - avail = min_t(int, length - pos, PAGE_SIZE); - if (actor->page[i]) { - pageaddr = kmap_atomic(actor->page[i]); - memcpy(buf + pos, pageaddr, avail); - kunmap_atomic(pageaddr); - } - pos += avail; - } -} - -void squashfs_buf_to_actor(void *buf, struct squashfs_page_actor *actor, - int length) -{ - void *pageaddr; - int pos = 0, avail, i; - - for (i = 0; i < actor->pages && pos < length; ++i) { - avail = min_t(int, length - pos, PAGE_SIZE); - if (actor->page[i]) { - pageaddr = kmap_atomic(actor->page[i]); - memcpy(pageaddr, buf + pos, avail); - kunmap_atomic(pageaddr); - } - pos += avail; - } -} - -void squashfs_bh_to_actor(struct buffer_head **bh, int nr_buffers, - struct squashfs_page_actor *actor, int offset, int length, int blksz) -{ - void *kaddr = NULL; - int bytes = 0, pgoff = 0, b = 0, p = 0, avail, i; - - while (bytes < length) { - if (actor->page[p]) { - kaddr = kmap_atomic(actor->page[p]); - while (pgoff < PAGE_SIZE && bytes < length) { - avail = min_t(int, blksz - offset, - PAGE_SIZE - pgoff); - memcpy(kaddr + pgoff, bh[b]->b_data + offset, - avail); - pgoff += avail; - bytes += avail; - offset = (offset + avail) % blksz; - if (!offset) { - put_bh(bh[b]); - ++b; - } - } - kunmap_atomic(kaddr); - pgoff = 0; - } else { - for (i = 0; i < PAGE_SIZE / blksz; ++i) { - if (bh[b]) - put_bh(bh[b]); - ++b; - } - bytes += PAGE_SIZE; - } - ++p; - } -} - -void squashfs_bh_to_buf(struct buffer_head **bh, int nr_buffers, void *buf, - int offset, int length, int blksz) -{ - int i, avail, bytes = 0; - - for (i = 0; i < nr_buffers && bytes < length; ++i) { - avail = min_t(int, length - bytes, blksz - offset); - if (bh[i]) { - memcpy(buf + bytes, bh[i]->b_data + offset, avail); - put_bh(bh[i]); - } - bytes += avail; - offset = 0; - } -} - -void free_page_array(struct page **page, int nr_pages) -{ - int i; - - for (i = 0; i < nr_pages; ++i) - __free_page(page[i]); - kfree(page); -} - -struct page **alloc_page_array(int nr_pages, int gfp_mask) -{ - int i; - struct page **page; - - page = kcalloc(nr_pages, sizeof(struct page *), gfp_mask); - if (!page) - return NULL; - for (i = 0; i < nr_pages; ++i) { - page[i] = alloc_page(gfp_mask); - if (!page[i]) { - free_page_array(page, i); - return NULL; - } - } - return page; -} diff --git a/fs/squashfs/page_actor.h b/fs/squashfs/page_actor.h index aa1ed790b5a3..98537eab27e2 100644 --- a/fs/squashfs/page_actor.h +++ b/fs/squashfs/page_actor.h @@ -5,61 +5,77 @@ * Phillip Lougher * * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level squashfsory. + * the COPYING file in the top-level directory. */ +#ifndef CONFIG_SQUASHFS_FILE_DIRECT struct squashfs_page_actor { - struct page **page; - void *pageaddr; + void **page; int pages; int length; int next_page; - void (*release_pages)(struct page **, int, int); }; -extern struct squashfs_page_actor *squashfs_page_actor_init(struct page **, - int, int, void (*)(struct page **, int, int)); -extern void squashfs_page_actor_free(struct squashfs_page_actor *, int); +static inline struct squashfs_page_actor *squashfs_page_actor_init(void **page, + int pages, int length) +{ + struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL); -extern void squashfs_actor_to_buf(struct squashfs_page_actor *, void *, int); -extern void squashfs_buf_to_actor(void *, struct squashfs_page_actor *, int); -extern void squashfs_bh_to_actor(struct buffer_head **, int, - struct squashfs_page_actor *, int, int, int); -extern void squashfs_bh_to_buf(struct buffer_head **, int, void *, int, int, - int); + if (actor == NULL) + return NULL; + + actor->length = length ? : pages * PAGE_SIZE; + actor->page = page; + actor->pages = pages; + actor->next_page = 0; + return actor; +} -/* - * Calling code should avoid sleeping between calls to squashfs_first_page() - * and squashfs_finish_page(). - */ static inline void *squashfs_first_page(struct squashfs_page_actor *actor) { actor->next_page = 1; - return actor->pageaddr = actor->page[0] ? kmap_atomic(actor->page[0]) - : NULL; + return actor->page[0]; } static inline void *squashfs_next_page(struct squashfs_page_actor *actor) { - if (!IS_ERR_OR_NULL(actor->pageaddr)) - kunmap_atomic(actor->pageaddr); - - if (actor->next_page == actor->pages) - return actor->pageaddr = ERR_PTR(-ENODATA); - - actor->pageaddr = actor->page[actor->next_page] ? - kmap_atomic(actor->page[actor->next_page]) : NULL; - ++actor->next_page; - return actor->pageaddr; + return actor->next_page == actor->pages ? NULL : + actor->page[actor->next_page++]; } static inline void squashfs_finish_page(struct squashfs_page_actor *actor) { - if (!IS_ERR_OR_NULL(actor->pageaddr)) - kunmap_atomic(actor->pageaddr); + /* empty */ } +#else +struct squashfs_page_actor { + union { + void **buffer; + struct page **page; + }; + void *pageaddr; + void *(*squashfs_first_page)(struct squashfs_page_actor *); + void *(*squashfs_next_page)(struct squashfs_page_actor *); + void (*squashfs_finish_page)(struct squashfs_page_actor *); + int pages; + int length; + int next_page; +}; -extern struct page **alloc_page_array(int, int); -extern void free_page_array(struct page **, int); - +extern struct squashfs_page_actor *squashfs_page_actor_init(void **, int, int); +extern struct squashfs_page_actor *squashfs_page_actor_init_special(struct page + **, int, int); +static inline void *squashfs_first_page(struct squashfs_page_actor *actor) +{ + return actor->squashfs_first_page(actor); +} +static inline void *squashfs_next_page(struct squashfs_page_actor *actor) +{ + return actor->squashfs_next_page(actor); +} +static inline void squashfs_finish_page(struct squashfs_page_actor *actor) +{ + actor->squashfs_finish_page(actor); +} +#endif #endif diff --git a/fs/squashfs/squashfs.h b/fs/squashfs/squashfs.h index f4faab52a879..887d6d270080 100644 --- a/fs/squashfs/squashfs.h +++ b/fs/squashfs/squashfs.h @@ -28,12 +28,8 @@ #define WARNING(s, args...) pr_warn("SQUASHFS: "s, ## args) /* block.c */ -extern int squashfs_init_read_wq(void); -extern void squashfs_destroy_read_wq(void); extern int squashfs_read_data(struct super_block *, u64, int, u64 *, struct squashfs_page_actor *); -extern int squashfs_read_data_async(struct super_block *, u64, int, u64 *, - struct squashfs_page_actor *); /* cache.c */ extern struct squashfs_cache *squashfs_cache_init(char *, int, int); @@ -74,9 +70,8 @@ extern __le64 *squashfs_read_fragment_index_table(struct super_block *, void squashfs_copy_cache(struct page *, struct squashfs_cache_entry *, int, int); -/* file_direct.c */ -extern int squashfs_readpages_block(struct page *, struct list_head *, - unsigned int *, struct address_space *, int, u64, int); +/* file_xxx.c */ +extern int squashfs_readpage_block(struct page *, u64, int); /* id.c */ extern int squashfs_get_id(struct super_block *, unsigned int, unsigned int *); diff --git a/fs/squashfs/squashfs_fs_sb.h b/fs/squashfs/squashfs_fs_sb.h index 8a6995de0277..ef69c31947bf 100644 --- a/fs/squashfs/squashfs_fs_sb.h +++ b/fs/squashfs/squashfs_fs_sb.h @@ -49,7 +49,7 @@ struct squashfs_cache_entry { int num_waiters; wait_queue_head_t wait_queue; struct squashfs_cache *cache; - struct page **page; + void **data; struct squashfs_page_actor *actor; }; @@ -75,6 +75,7 @@ struct squashfs_sb_info { unsigned short block_log; long long bytes_used; unsigned int inodes; + unsigned int fragments; int xattr_ids; }; #endif diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c index e2a0a7342bf8..1516bb779b8d 100644 --- a/fs/squashfs/super.c +++ b/fs/squashfs/super.c @@ -175,6 +175,7 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent) msblk->inode_table = le64_to_cpu(sblk->inode_table_start); msblk->directory_table = le64_to_cpu(sblk->directory_table_start); msblk->inodes = le32_to_cpu(sblk->inodes); + msblk->fragments = le32_to_cpu(sblk->fragments); flags = le16_to_cpu(sblk->flags); TRACE("Found valid superblock on %pg\n", sb->s_bdev); @@ -185,7 +186,7 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent) TRACE("Filesystem size %lld bytes\n", msblk->bytes_used); TRACE("Block size %d\n", msblk->block_size); TRACE("Number of inodes %d\n", msblk->inodes); - TRACE("Number of fragments %d\n", le32_to_cpu(sblk->fragments)); + TRACE("Number of fragments %d\n", msblk->fragments); TRACE("Number of ids %d\n", le16_to_cpu(sblk->no_ids)); TRACE("sblk->inode_table_start %llx\n", msblk->inode_table); TRACE("sblk->directory_table_start %llx\n", msblk->directory_table); @@ -272,7 +273,7 @@ allocate_id_index_table: sb->s_export_op = &squashfs_export_ops; handle_fragments: - fragments = le32_to_cpu(sblk->fragments); + fragments = msblk->fragments; if (fragments == 0) goto check_directory_table; @@ -444,15 +445,9 @@ static int __init init_squashfs_fs(void) if (err) return err; - if (!squashfs_init_read_wq()) { - destroy_inodecache(); - return -ENOMEM; - } - err = register_filesystem(&squashfs_fs_type); if (err) { destroy_inodecache(); - squashfs_destroy_read_wq(); return err; } @@ -466,7 +461,6 @@ static void __exit exit_squashfs_fs(void) { unregister_filesystem(&squashfs_fs_type); destroy_inodecache(); - squashfs_destroy_read_wq(); } diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c index 2f7be1fb167c..6bfaef73d065 100644 --- a/fs/squashfs/xz_wrapper.c +++ b/fs/squashfs/xz_wrapper.c @@ -55,7 +55,7 @@ static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk, struct comp_opts *opts; int err = 0, n; - opts = kmalloc(sizeof(*opts), GFP_ATOMIC); + opts = kmalloc(sizeof(*opts), GFP_KERNEL); if (opts == NULL) { err = -ENOMEM; goto out2; @@ -136,7 +136,6 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, enum xz_ret xz_err; int avail, total = 0, k = 0; struct squashfs_xz *stream = strm; - void *buf = NULL; xz_dec_reset(stream->state); stream->buf.in_pos = 0; @@ -157,20 +156,12 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, if (stream->buf.out_pos == stream->buf.out_size) { stream->buf.out = squashfs_next_page(output); - if (!IS_ERR(stream->buf.out)) { + if (stream->buf.out != NULL) { stream->buf.out_pos = 0; total += PAGE_SIZE; } } - if (!stream->buf.out) { - if (!buf) { - buf = kmalloc(PAGE_SIZE, GFP_ATOMIC); - if (!buf) - goto out; - } - stream->buf.out = buf; - } xz_err = xz_dec_run(stream->state, &stream->buf); if (stream->buf.in_pos == stream->buf.in_size && k < b) @@ -182,13 +173,11 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, if (xz_err != XZ_STREAM_END || k < b) goto out; - kfree(buf); return total + stream->buf.out_pos; out: for (; k < b; k++) put_bh(bh[k]); - kfree(buf); return -EIO; } diff --git a/fs/squashfs/zlib_wrapper.c b/fs/squashfs/zlib_wrapper.c index d917c728422b..2ec24d128bce 100644 --- a/fs/squashfs/zlib_wrapper.c +++ b/fs/squashfs/zlib_wrapper.c @@ -66,7 +66,6 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, struct buffer_head **bh, int b, int offset, int length, struct squashfs_page_actor *output) { - void *buf = NULL; int zlib_err, zlib_init = 0, k = 0; z_stream *stream = strm; @@ -85,19 +84,10 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, if (stream->avail_out == 0) { stream->next_out = squashfs_next_page(output); - if (!IS_ERR(stream->next_out)) + if (stream->next_out != NULL) stream->avail_out = PAGE_SIZE; } - if (!stream->next_out) { - if (!buf) { - buf = kmalloc(PAGE_SIZE, GFP_ATOMIC); - if (!buf) - goto out; - } - stream->next_out = buf; - } - if (!zlib_init) { zlib_err = zlib_inflateInit(stream); if (zlib_err != Z_OK) { @@ -125,13 +115,11 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, if (k < b) goto out; - kfree(buf); return stream->total_out; out: for (; k < b; k++) put_bh(bh[k]); - kfree(buf); return -EIO; } diff --git a/include/linux/sched.h b/include/linux/sched.h index fe74de4a2c6c..18c722dd75f9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -3154,7 +3154,6 @@ static inline void set_task_comm(struct task_struct *tsk, const char *from) } extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk); #define get_task_comm(buf, tsk) ({ \ - BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN); \ __get_task_comm(buf, sizeof(buf), tsk); \ }) diff --git a/include/net/tcp.h b/include/net/tcp.h index 58d3e96eddaa..8d4ad17bac3b 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -364,7 +364,7 @@ ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); -void tcp_enter_quickack_mode(struct sock *sk); +void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks); static inline void tcp_dec_quickack_mode(struct sock *sk, const unsigned int pkts) { diff --git a/kernel/sched/swait.c b/kernel/sched/swait.c index 82f0dff90030..9c2da06a8869 100644 --- a/kernel/sched/swait.c +++ b/kernel/sched/swait.c @@ -33,9 +33,6 @@ void swake_up(struct swait_queue_head *q) { unsigned long flags; - if (!swait_active(q)) - return; - raw_spin_lock_irqsave(&q->lock, flags); swake_up_locked(q); raw_spin_unlock_irqrestore(&q->lock, flags); @@ -51,9 +48,6 @@ void swake_up_all(struct swait_queue_head *q) struct swait_queue *curr; LIST_HEAD(tmp); - if (!swait_active(q)) - return; - raw_spin_lock_irq(&q->lock); list_splice_init(&q->task_list, &tmp); while (!list_empty(&tmp)) { diff --git a/net/dsa/slave.c b/net/dsa/slave.c index 5000e6f20f4a..339d9c678d3e 100644 --- a/net/dsa/slave.c +++ b/net/dsa/slave.c @@ -1199,6 +1199,9 @@ int dsa_slave_suspend(struct net_device *slave_dev) { struct dsa_slave_priv *p = netdev_priv(slave_dev); + if (!netif_running(slave_dev)) + return 0; + netif_device_detach(slave_dev); if (p->phy) { @@ -1216,6 +1219,9 @@ int dsa_slave_resume(struct net_device *slave_dev) { struct dsa_slave_priv *p = netdev_priv(slave_dev); + if (!netif_running(slave_dev)) + return 0; + netif_device_attach(slave_dev); if (p->phy) { diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 9be03120d84c..78ee2fcd3699 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -282,19 +282,19 @@ __be32 fib_compute_spec_dst(struct sk_buff *skb) return ip_hdr(skb)->daddr; in_dev = __in_dev_get_rcu(dev); - BUG_ON(!in_dev); net = dev_net(dev); scope = RT_SCOPE_UNIVERSE; if (!ipv4_is_zeronet(ip_hdr(skb)->saddr)) { + bool vmark = in_dev && IN_DEV_SRC_VMARK(in_dev); struct flowi4 fl4 = { .flowi4_iif = LOOPBACK_IFINDEX, .flowi4_oif = l3mdev_master_ifindex_rcu(dev), .daddr = ip_hdr(skb)->saddr, .flowi4_tos = RT_TOS(ip_hdr(skb)->tos), .flowi4_scope = scope, - .flowi4_mark = IN_DEV_SRC_VMARK(in_dev) ? skb->mark : 0, + .flowi4_mark = vmark ? skb->mark : 0, }; if (!fib_lookup(net, &fl4, &res, 0)) return FIB_RES_PREFSRC(net, res); diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 9169859506b7..7e44d23b0328 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -324,6 +324,10 @@ static u32 bbr_target_cwnd(struct sock *sk, u32 bw, int gain) /* Reduce delayed ACKs by rounding up cwnd to the next even number. */ cwnd = (cwnd + 1) & ~1U; + /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ + if (bbr->mode == BBR_PROBE_BW && gain > BBR_UNIT) + cwnd += 2; + return cwnd; } diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c index dd52ccb812ea..8905a0aec8ee 100644 --- a/net/ipv4/tcp_dctcp.c +++ b/net/ipv4/tcp_dctcp.c @@ -138,7 +138,7 @@ static void dctcp_ce_state_0_to_1(struct sock *sk) */ if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) __tcp_send_ack(sk, ca->prior_rcv_nxt); - tcp_enter_quickack_mode(sk); + tcp_enter_quickack_mode(sk, 1); } ca->prior_rcv_nxt = tp->rcv_nxt; @@ -159,7 +159,7 @@ static void dctcp_ce_state_1_to_0(struct sock *sk) */ if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) __tcp_send_ack(sk, ca->prior_rcv_nxt); - tcp_enter_quickack_mode(sk); + tcp_enter_quickack_mode(sk, 1); } ca->prior_rcv_nxt = tp->rcv_nxt; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 484d7bc6fed6..16baf3b29223 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -199,21 +199,23 @@ static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb) } } -static void tcp_incr_quickack(struct sock *sk) +static void tcp_incr_quickack(struct sock *sk, unsigned int max_quickacks) { struct inet_connection_sock *icsk = inet_csk(sk); unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss); if (quickacks == 0) quickacks = 2; + quickacks = min(quickacks, max_quickacks); if (quickacks > icsk->icsk_ack.quick) - icsk->icsk_ack.quick = min(quickacks, TCP_MAX_QUICKACKS); + icsk->icsk_ack.quick = quickacks; } -void tcp_enter_quickack_mode(struct sock *sk) +void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks) { struct inet_connection_sock *icsk = inet_csk(sk); - tcp_incr_quickack(sk); + + tcp_incr_quickack(sk, max_quickacks); icsk->icsk_ack.pingpong = 0; icsk->icsk_ack.ato = TCP_ATO_MIN; } @@ -249,8 +251,10 @@ static void tcp_ecn_withdraw_cwr(struct tcp_sock *tp) tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR; } -static void __tcp_ecn_check_ce(struct tcp_sock *tp, const struct sk_buff *skb) +static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) { + struct tcp_sock *tp = tcp_sk(sk); + switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) { case INET_ECN_NOT_ECT: /* Funny extension: if ECT is not set on a segment, @@ -258,31 +262,31 @@ static void __tcp_ecn_check_ce(struct tcp_sock *tp, const struct sk_buff *skb) * it is probably a retransmit. */ if (tp->ecn_flags & TCP_ECN_SEEN) - tcp_enter_quickack_mode((struct sock *)tp); + tcp_enter_quickack_mode(sk, 2); break; case INET_ECN_CE: - if (tcp_ca_needs_ecn((struct sock *)tp)) - tcp_ca_event((struct sock *)tp, CA_EVENT_ECN_IS_CE); + if (tcp_ca_needs_ecn(sk)) + tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { /* Better not delay acks, sender can have a very low cwnd */ - tcp_enter_quickack_mode((struct sock *)tp); + tcp_enter_quickack_mode(sk, 2); tp->ecn_flags |= TCP_ECN_DEMAND_CWR; } tp->ecn_flags |= TCP_ECN_SEEN; break; default: - if (tcp_ca_needs_ecn((struct sock *)tp)) - tcp_ca_event((struct sock *)tp, CA_EVENT_ECN_NO_CE); + if (tcp_ca_needs_ecn(sk)) + tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); tp->ecn_flags |= TCP_ECN_SEEN; break; } } -static void tcp_ecn_check_ce(struct tcp_sock *tp, const struct sk_buff *skb) +static void tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) { - if (tp->ecn_flags & TCP_ECN_OK) - __tcp_ecn_check_ce(tp, skb); + if (tcp_sk(sk)->ecn_flags & TCP_ECN_OK) + __tcp_ecn_check_ce(sk, skb); } static void tcp_ecn_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th) @@ -676,7 +680,7 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) /* The _first_ data packet received, initialize * delayed ACK engine. */ - tcp_incr_quickack(sk); + tcp_incr_quickack(sk, TCP_MAX_QUICKACKS); icsk->icsk_ack.ato = TCP_ATO_MIN; } else { int m = now - icsk->icsk_ack.lrcvtime; @@ -692,13 +696,13 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) /* Too long gap. Apparently sender failed to * restart window, so that we send ACKs quickly. */ - tcp_incr_quickack(sk); + tcp_incr_quickack(sk, TCP_MAX_QUICKACKS); sk_mem_reclaim(sk); } } icsk->icsk_ack.lrcvtime = now; - tcp_ecn_check_ce(tp, skb); + tcp_ecn_check_ce(sk, skb); if (skb->len >= 128) tcp_grow_window(sk, skb); @@ -4211,7 +4215,7 @@ static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb) if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq && before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) { NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST); - tcp_enter_quickack_mode(sk); + tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); if (tcp_is_sack(tp) && sysctl_tcp_dsack) { u32 end_seq = TCP_SKB_CB(skb)->end_seq; @@ -4455,7 +4459,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) u32 seq, end_seq; bool fragstolen; - tcp_ecn_check_ce(tp, skb); + tcp_ecn_check_ce(sk, skb); if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP); @@ -4735,7 +4739,7 @@ queue_and_out: tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); out_of_window: - tcp_enter_quickack_mode(sk); + tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); inet_csk_schedule_ack(sk); drop: tcp_drop(sk, skb); @@ -4746,8 +4750,6 @@ drop: if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt + tcp_receive_window(tp))) goto out_of_window; - tcp_enter_quickack_mode(sk); - if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) { /* Partial packet, seq < rcv_next < end_seq */ SOCK_DEBUG(sk, "partial packet: rcv_next %X seq %X - %X\n", @@ -5831,7 +5833,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, * to stand against the temptation 8) --ANK */ inet_csk_schedule_ack(sk); - tcp_enter_quickack_mode(sk); + tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, TCP_DELACK_MAX, TCP_RTO_MAX); diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 15e6e7b9fd2b..8d0aafbdbbc3 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -62,6 +62,7 @@ #include #include #include +#include #include #include @@ -654,6 +655,7 @@ static int netlink_create(struct net *net, struct socket *sock, int protocol, if (protocol < 0 || protocol >= MAX_LINKS) return -EPROTONOSUPPORT; + protocol = array_index_nospec(protocol, MAX_LINKS); netlink_lock_table(); #ifdef CONFIG_MODULES diff --git a/net/socket.c b/net/socket.c index 34dabf4b9192..ded440292bee 100644 --- a/net/socket.c +++ b/net/socket.c @@ -89,6 +89,7 @@ #include #include #include +#include #include #include @@ -2352,6 +2353,7 @@ SYSCALL_DEFINE2(socketcall, int, call, unsigned long __user *, args) if (call < 1 || call > SYS_SENDMMSG) return -EINVAL; + call = array_index_nospec(call, SYS_SENDMMSG + 1); len = nargs[call]; if (len > sizeof(a))